diff --git a/build_effective_set_generator/scripts/test_generate_effective_set.py b/build_effective_set_generator/scripts/test_generate_effective_set.py index 264b1734..c820d13c 100644 --- a/build_effective_set_generator/scripts/test_generate_effective_set.py +++ b/build_effective_set_generator/scripts/test_generate_effective_set.py @@ -34,7 +34,7 @@ def test_render_envs(cluster_name, env_name, sd_version): generated_dir = f"{g_test_dir}/{cluster_name}/{env_name}" fill_creds(generated_dir) generate_effective_set_for_env(cluster_name, env_name, g_test_dir) - files_to_compare = get_all_files_in_dir(source_dir, source_dir+"/") + files_to_compare = get_all_files_in_dir(source_dir) logger.info(dump_as_yaml_format(files_to_compare)) match, mismatch, errors = filecmp.cmpfiles(source_dir , generated_dir, files_to_compare, shallow=False) logger.info(f"Match: {dump_as_yaml_format(match)}") diff --git a/build_envgene/build/Dockerfile b/build_envgene/build/Dockerfile index 39bddfc2..36aad091 100644 --- a/build_envgene/build/Dockerfile +++ b/build_envgene/build/Dockerfile @@ -38,7 +38,6 @@ COPY build_envgene/scripts /module/scripts COPY creds_rotation/scripts /module/creds_rotation_scripts COPY build_* create_* produce_* sort* /build_env/ COPY scripts /build_env/scripts -COPY env-builder /build_env/env-builder COPY schemas /build_env/schemas ENV ANSIBLE_LIBRARY=/module/ansible/library diff --git a/build_envgene/build/requirements.txt b/build_envgene/build/requirements.txt index 4c82b098..47b770e6 100644 --- a/build_envgene/build/requirements.txt +++ b/build_envgene/build/requirements.txt @@ -16,6 +16,8 @@ termcolor==2.4.0 ansible-core==2.17.12 cffi==1.16.0 click==8.1.3 +deepmerge~=2.0 +cp==2020.12.3 # Additional required packages platformdirs>=3.0.0 diff --git a/dependencies/tests_requirements.txt b/dependencies/tests_requirements.txt index 0d6fe460..999f8245 100644 --- a/dependencies/tests_requirements.txt +++ b/dependencies/tests_requirements.txt @@ -29,3 +29,5 @@ pytest==7.4.3 junitparser==3.1.2 hiyapyco==0.6.0 click==8.1.7 +deepmerge~=2.0 +cp==2020.12.3 diff --git a/env-builder/ansible.cfg b/env-builder/ansible.cfg deleted file mode 100644 index 8e2b94db..00000000 --- a/env-builder/ansible.cfg +++ /dev/null @@ -1,23 +0,0 @@ -[defaults] -#log_path=./logs/ansible.log -host_key_checking = False -retry_files_enabled = False -callback_whitelist = timer,profile_roles - - -roles_path = roles -inventory = hosts -#vault_password_file = .current-deployment/.vault-pass - -# YAML callback plugin -stdout_callback = yaml -bin_ansible_callbacks = True - - - -timeout=30 -force_color = 1 - -[ssh_connection] -ssh_args='-o ControlMaster=auto -o ControlPersist=10m -o ServerAliveCountMax=15 -o ServerAliveInterval=15' - diff --git a/env-builder/main.yaml b/env-builder/main.yaml deleted file mode 100644 index 8ecacb11..00000000 --- a/env-builder/main.yaml +++ /dev/null @@ -1,266 +0,0 @@ ---- -- name: Generate jobs set - hosts: 127.0.0.1 - connection: local - gather_facts: false - tasks: - #get env variables - - name: Get input arguments - ansible.builtin.set_fact: - env: "{{ lookup('ansible.builtin.env', 'env') }}" - cluster_name: "{{ lookup('ansible.builtin.env', 'cluster_name') }}" - templates_dir: "{{ lookup('ansible.builtin.env', 'templates_dir') }}" - env_instances_dir: "{{ lookup('ansible.builtin.env', 'env_instances_dir') }}" - render_dir: "{{ lookup('ansible.builtin.env', 'render_dir') }}" - template_version: "{{ lookup('ansible.builtin.env', 'template_version') }}" - cloud_passport_file_path: "{{ lookup('ansible.builtin.env', 'cloud_passport_file_path') }}" - cmdb_url: "{{ lookup('ansible.builtin.env', 'cmdb_url') }}" - output_dir: "{{ lookup('ansible.builtin.env', 'output_dir') }}" - render_parameters_dir: "{{ lookup('ansible.builtin.env', 'render_parameters_dir') }}" - - name: Print arguments - debug: - msg: "Building environment {{ env }} from templates dir {{ templates_dir }} and instances dir {{ env_instances_dir }}" - - #get env variables - - name: Get inventory - ansible.builtin.set_fact: - env_definition: "{{ lookup('file', '{{ env_instances_dir }}/Inventory/env_definition.yml') | from_yaml }}" - - - name: Print inventory - debug: - var: env_definition - - #get env variables - - name: Get cloud passport - ansible.builtin.set_fact: - cloud_passport: "{{ lookup('file', '{{ cloud_passport_file_path }}') | from_yaml }}" - when: cloud_passport_file_path | trim != '' - - #generate env_config - - name: Generate config for env {{ env }} - ansible.builtin.set_fact: - config: "{{ lookup('template', 'env_config.yml.j2') | from_yaml }}" - - - name: Print config - debug: - var: config - - - name: Set current env - ansible.builtin.set_fact: - current_env: "{{ config.environment }}" - - - name: Set current vars - ansible.builtin.set_fact: - cloud: "{{ current_env.cloud }}" - tenant: "{{ current_env.tenant | default('')}}" - deployer: "{{ current_env.deployer | default('')}}" - - - name: print current env - debug: - msg: "current_env: {{ current_env }}" - - - name: Set Gitlab ENV parameters - ansible.builtin.set_fact: - ND_CMDB_CONFIG_REF: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') | default('No SHA') }}" - ND_CMDB_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME') | default('No Ref Name') }}" - ND_CMDB_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG') | default('No Ref tag') }}" - ND_CDMB_REPOSITORY_URL: "{{ lookup('ansible.builtin.env', 'CI_REPOSITORY_URL') | default('No Ref URL') }}" - ND_CMDB_ENV_TEMPLATE: "{{ env_template | default(current_env.env_template) }}" - - - name: Set current_env_dir - ansible.builtin.set_fact: - current_env_dir: "{{ render_dir }}/{{ env }}" - - - name: Set Env Template - ansible.builtin.include_vars: - file: "{{ templates_dir }}/env_templates/{{current_env.env_template}}.yaml" - name: current_env_template - - - name: Generate solution_structure variable - ansible.builtin.include_role: - name: generate_solution_structure - vars: - _template_name: "{{ current_env.env_template }}" - _tenant: "{{ tenant }}" - _env: "{{ env }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - - - name: Gen tenant - ansible.builtin.include_role: - name: generate_tenant - vars: - _env: "{{ env }}" - _tenant: "{{ tenant }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - _template: "{{current_env_template.tenant}}" - - - name: Gen cloud - ansible.builtin.include_role: - name: generate_cloud - vars: - _env: "{{ env }}" - _tenant: "{{ tenant }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - _template: "{{ current_env_template.cloud }}" - - - name: Gen namespaces - ansible.builtin.include_role: - name: generate_namespaces - vars: - _env: "{{ env }}" - _tenant: "{{ tenant }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - loop: "{{ current_env_template.namespaces }}" - loop_control: - loop_var: namespace - - - name: Gen resource profiles - ansible.builtin.include_role: - name: generate_profiles - vars: - _env: "{{ env }}" - _tenant: "{{ tenant }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - loop: "{{ current_env_template.resourceProfiles }}" - loop_control: - loop_var: profile - when: current_env_template.resourceProfiles is defined - - - name: Gen composite structure - ansible.builtin.include_role: - name: generate_composite_structure - vars: - _env: "{{ env }}" - _tenant: "{{ tenant }}" - _cloud: "{{ current_env.cloud }}" - _current_env_dir: "{{ current_env_dir }}" - _template: "{{current_env_template.composite_structure}}" - when: current_env_template.composite_structure is defined - - - name: Generate environment specific schema - ansible.builtin.copy: - src: "{{ current_env_template.envSpecificSchema }}" - dest: "{{ current_env_dir }}/env-specific-schema.yml" - when: current_env_template.envSpecificSchema is defined - - - name: Find parametersets with Jinja templates - ansible.builtin.find: - paths: "{{ render_parameters_dir }}" - patterns: '*.yml.j2,*.yaml.j2' - recurse: true - register: paramset_templates - - - name: Generate paramsets defined using Jinja templates - ansible.builtin.include_role: - name: generate_paramsets - vars: - _paramset_path: "{{ paramset_file.path }}" - loop: "{{ paramset_templates.files }}" - loop_control: - loop_var: paramset_file - when: paramset_templates.files | length > 0 - -# Validate Required Variables - - - name: Validate required variables - ansible.builtin.assert: - that: - - templates_dir is defined - - env_instances_dir is defined - - cluster_name is defined - - current_env_dir is defined - fail_msg: "Missing one or more required variables: templates_dir, env_instances_dir, cluster_name, current_env_dir. Please check your configuration." - -# Check for APP_REG_DEFS_JOB Parameter - - - name: Check if APP_REG_DEFS_JOB is defined - ansible.builtin.set_fact: - use_external_defs: "{{ app_reg_defs_job is defined and app_reg_defs_job | length > 0 }}" - - - name: Log external definitions usage - ansible.builtin.debug: - msg: "APP_REG_DEFS_JOB variable is set, Application and Registry definitions from corresponding job will be used for the Environment" - when: use_external_defs - - - name: Process external definitions - block: - - name: Get and unpack external definitions - ansible.builtin.unarchive: - src: "{{ jenkins_artifacts_dir }}/{{ app_reg_defs_job }}/definitions.zip" - dest: "/tmp/external_defs" - remote_src: true - - - name: Copy external AppDefs to environment - ansible.builtin.copy: - src: "/tmp/external_defs/{{ app_defs_path }}/" - dest: "{{ current_env_dir }}/AppDefs/" - remote_src: true - when: app_defs_path is defined - - - name: Copy external RegDefs to environment - ansible.builtin.copy: - src: "/tmp/external_defs/{{ reg_defs_path }}/" - dest: "{{ current_env_dir }}/RegDefs/" - remote_src: true - when: reg_defs_path is defined - when: use_external_defs - -# Discover and Render AppDef / RegDef Templates - - - name: Find AppDef templates - ansible.builtin.find: - paths: "{{ templates_dir }}/appdefs" - patterns: - - "*.yaml.j2" - - "*.yml.j2" - - "*.j2" - - "*.yaml" - - "*.yml" - recurse: true - use_regex: false - file_type: file - register: appdef_templates - ignore_errors: true - when: not use_external_defs - - - name: Find RegDef templates - ansible.builtin.find: - paths: "{{ templates_dir }}/regdefs" - patterns: - - "*.yaml.j2" - - "*.yml.j2" - - "*.j2" - - "*.yaml" - - "*.yml" - recurse: true - use_regex: false - file_type: file - register: regdef_templates - ignore_errors: true - when: not use_external_defs - - - name: Debug - Found AppDef and RegDef templates - ansible.builtin.debug: - msg: - - "AppDefs Found: {{ appdef_templates.files | default([]) | length }}" - - "RegDefs Found: {{ regdef_templates.files | default([]) | length }}" - when: not use_external_defs - -# Conditionally Render Templates if Present - - - name: Render AppDef / RegDef templates - ansible.builtin.include_role: - name: generate_appregdefs - vars: - appdef_templates_files: "{{ appdef_templates.files | default([]) }}" - regdef_templates_files: "{{ regdef_templates.files | default([]) }}" - when: > - not use_external_defs and - ((appdef_templates.files is defined and appdef_templates.files | length > 0) or - (regdef_templates.files is defined and regdef_templates.files | length > 0)) - diff --git a/env-builder/roles/generate_appregdefs/tasks/main.yaml b/env-builder/roles/generate_appregdefs/tasks/main.yaml deleted file mode 100644 index e0e622a4..00000000 --- a/env-builder/roles/generate_appregdefs/tasks/main.yaml +++ /dev/null @@ -1,99 +0,0 @@ ---- -- name: Validate mandatory variables - assert: - that: - - current_env_dir is defined - - cluster_name is defined - - env_instances_dir is defined - - (appdef_templates.files | length) >= 0 - - (regdef_templates.files | length) >= 0 - -- name: Ensure AppDefs directory exists - file: - path: "{{ current_env_dir }}/AppDefs" - state: directory - mode: '0755' - when: appdef_templates.files | length > 0 - -- name: Ensure RegDefs directory exists - file: - path: "{{ current_env_dir }}/RegDefs" - state: directory - mode: '0755' - when: regdef_templates.files | length > 0 - -- name: Set config file candidates - set_fact: - cluster_config_yaml: "{{ output_dir }}/{{ cluster_name }}/configuration/appregdef_config.yaml" - cluster_config_yml: "{{ output_dir }}/{{ cluster_name }}/configuration/appregdef_config.yml" - global_config_yaml: "{{ output_dir | dirname }}/configuration/appregdef_config.yaml" - global_config_yml: "{{ output_dir | dirname }}/configuration/appregdef_config.yml" - -- name: Stat potential config files - stat: - path: "{{ item }}" - loop: - - "{{ cluster_config_yaml }}" - - "{{ cluster_config_yml }}" - - "{{ global_config_yaml }}" - - "{{ global_config_yml }}" - loop_control: - label: "{{ item | basename }}" - register: config_stats - -- name: Pick config path by existence - set_fact: - appregdef_config_path: >- - {{ ( - config_stats.results[0].stat.exists and cluster_config_yaml - ) or ( - config_stats.results[1].stat.exists and cluster_config_yml - ) or ( - config_stats.results[2].stat.exists and global_config_yaml - ) or ( - config_stats.results[3].stat.exists and global_config_yml - ) or 'none' - }} - -- name: Load overrides config (if any) - set_fact: - appregdef_config: "{{ lookup('file', appregdef_config_path) | from_yaml }}" - when: appregdef_config_path != 'none' - ignore_errors: yes - -- name: Default to empty config - set_fact: - appregdef_config: {} - when: appregdef_config is not defined - -- name: Extract override mappings - set_fact: - appdefs_overrides: "{{ appregdef_config.appdefs.overrides | default({}) }}" - regdefs_overrides: "{{ appregdef_config.regdefs.overrides | default({}) }}" - -- name: Prepare regdefs dict for Jinja context - set_fact: - regdefs: - overrides: "{{ regdefs_overrides }}" - -- name: Timestamp for unique filenames - set_fact: - render_timestamp: "{{ lookup('pipe', 'date +%s') }}" - -- name: Render all AppDefs - include_tasks: render_single_appdef.yaml - loop: "{{ appdef_templates.files }}" - loop_control: - label: "{{ item.path | basename }}" - vars: - appdef_item: "{{ item }}" - when: appdef_templates.files | length > 0 - -- name: Render all RegDefs - include_tasks: render_single_regdef.yaml - loop: "{{ regdef_templates.files }}" - loop_control: - label: "{{ item.path | basename }}" - vars: - regdef_item: "{{ item }}" - when: regdef_templates.files | length > 0 diff --git a/env-builder/roles/generate_appregdefs/tasks/render_single_appdef.yaml b/env-builder/roles/generate_appregdefs/tasks/render_single_appdef.yaml deleted file mode 100644 index 0d4e2ec5..00000000 --- a/env-builder/roles/generate_appregdefs/tasks/render_single_appdef.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# roles/generate_appregdefs/tasks/render_single_appdef.yaml -- block: - - name: Ensure tmp_render_dir and render_timestamp are set - set_fact: - tmp_render_dir: "{{ tmp_render_dir | default('/tmp') }}" - render_timestamp: >- - {{ render_timestamp | default( - (ansible_date_time.epoch - if (ansible_date_time is defined and ansible_date_time.epoch is defined) - else lookup('pipe','date +%s') - ) - ) - }} - - # Prepare overrides for Jinja - - name: Prepare appdefs dict for Jinja context - set_fact: - appdefs: - overrides: "{{ appdefs_overrides }}" - - # Load the template as raw text - - name: Load raw AppDef template via file lookup - set_fact: - appdef_raw: "{{ lookup('ansible.builtin.file', item.path) }}" - - # Extract name, artifactId, groupId into a dict - - name: Extract metadata (name, artifactId, groupId) into dict - set_fact: - appdef_meta: "{{ dict(appdef_raw | regex_findall('^\\s*(name|artifactId|groupId):\\s*\"([^\\\"]+)\"', multiline=True)) }}" - - # Fail fast if name is missing - - name: Ensure AppDef name was parsed - assert: - that: - - appdef_meta.name is defined - - appdef_meta.name | length > 0 - fail_msg: "Template {{ item.path }} is missing name metadata" - - # Render with Jinja into a temp file - - name: Render AppDef template using Jinja - template: - src: "{{ item.path }}" - dest: "{{ tmp_render_dir }}/{{ (item.path | basename | regex_replace('\\.j2$','')) }}.{{ render_timestamp }}.rendered.appdef.yml" - vars: - current_env: "{{ current_env }}" - appdefs: "{{ appdefs }}" - artifactId: "{{ appdef_meta.artifactId }}" - groupId: "{{ appdef_meta.groupId }}" - app_lookup_key: "{{ appdef_meta.groupId ~ ':' ~ appdef_meta.artifactId }}" - - # Read and decode the rendered output - - name: Read rendered AppDef into string - slurp: - src: "{{ tmp_render_dir }}/{{ (item.path | basename | regex_replace('\\.j2$','')) }}.{{ render_timestamp }}.rendered.appdef.yml" - register: appdef_rendered_slurp - - - name: Decode rendered AppDef content - set_fact: - appdef_raw_rendered: "{{ appdef_rendered_slurp.content | b64decode }}" - - # Save final AppDef to the output dir - - name: Save final rendered AppDef - copy: - content: "{{ appdef_raw_rendered }}" - dest: "{{ current_env_dir }}/AppDefs/{{ appdef_meta.name }}.yml" - mode: '0644' diff --git a/env-builder/roles/generate_appregdefs/tasks/render_single_regdef.yaml b/env-builder/roles/generate_appregdefs/tasks/render_single_regdef.yaml deleted file mode 100644 index 114604a7..00000000 --- a/env-builder/roles/generate_appregdefs/tasks/render_single_regdef.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# roles/generate_appregdefs/tasks/render_single_regdef.yaml - -- block: - # 1) Ensure tmp_render_dir + timestamp (fact / shell fallback) - - name: Ensure tmp_render_dir and render_timestamp are set - set_fact: - tmp_render_dir: "{{ tmp_render_dir | default('/tmp') }}" - render_timestamp: >- - {{ render_timestamp | default( - (ansible_date_time.epoch - if (ansible_date_time is defined and ansible_date_time.epoch is defined) - else lookup('pipe','date +%s') - ) - ) - }} - - # 2) Prepare overrides for Jinja - - name: Prepare regdefs dict for Jinja context - set_fact: - regdefs: - overrides: "{{ regdefs_overrides }}" - - # 3) Compute a stable base name (strip trailing .j2) - - name: Extract template_name (basename without .j2) - set_fact: - template_name: "{{ item.path | basename | regex_replace('\\.j2$','') }}" - - # 4) Render into a timestamped temp file - - name: Render RegDef template using Jinja - template: - src: "{{ item.path }}" - dest: "{{ tmp_render_dir }}/{{ template_name }}.{{ render_timestamp }}.rendered.regdef.yml" - vars: - current_env: "{{ current_env }}" - regdefs: "{{ regdefs }}" - - # 5) Read & decode the rendered output - - name: Read rendered RegDef into string - slurp: - src: "{{ tmp_render_dir }}/{{ template_name }}.{{ render_timestamp }}.rendered.regdef.yml" - register: regdef_rendered_slurp - - - name: Decode rendered RegDef content - set_fact: - regdef_raw_rendered: "{{ regdef_rendered_slurp.content | b64decode }}" - - # 6) Parse for the name field - - name: Parse rendered RegDef YAML into dict - set_fact: - regdef_content: "{{ regdef_raw_rendered | from_yaml }}" - - # 7) Fail fast if name is missing - - name: Ensure RegDef name was parsed - assert: - that: - - regdef_content.name is defined - - regdef_content.name | length > 0 - fail_msg: "Rendered RegDef from {{ item.path }} is missing a name field" - - # 8) Write out the final file - - name: Save final rendered RegDef - copy: - content: "{{ regdef_raw_rendered }}" - dest: "{{ current_env_dir }}/RegDefs/{{ regdef_content.name }}.yml" - mode: '0644' diff --git a/env-builder/roles/generate_cloud/tasks/main.yaml b/env-builder/roles/generate_cloud/tasks/main.yaml deleted file mode 100644 index 8e506d26..00000000 --- a/env-builder/roles/generate_cloud/tasks/main.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Tell the cloud attribute format - set_fact: - is_old_format: "{{ _template | type_debug != 'dict' }}" - -- name: Generate Cloud yaml for cloud {{ _cloud }} - ansible.builtin.blockinfile: - path: "{{ current_env_dir }}/cloud.yml" - block: "{{ lookup('template', _template) }}" - create: yes - when: is_old_format - -- name: Generate Cloud yaml for cloud {{ _cloud }} using cloud.template_path value - ansible.builtin.blockinfile: - path: "{{ current_env_dir }}/cloud.yml" - block: "{{ lookup('template', _template.template_path) }}" - create: yes - when: not is_old_format - -- name: Generate override Cloud yaml for {{ _cloud }} - ansible.builtin.blockinfile: - path: "{{ _current_env_dir }}/cloud.yml_override" - block: "{{ _template.template_override | to_nice_yaml }}" - create: yes - when: - - _template.template_override is defined - - _template.template_override != '' - - not is_old_format - diff --git a/env-builder/roles/generate_composite_structure/tasks/main.yaml b/env-builder/roles/generate_composite_structure/tasks/main.yaml deleted file mode 100644 index 6969c46a..00000000 --- a/env-builder/roles/generate_composite_structure/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Generate Compsite Structure yaml {{ _composite_structure }} - ansible.builtin.blockinfile: - path: "{{ current_env_dir }}/composite_structure.yml" - block: "{{ lookup('template', _template) }}" - create: yes diff --git a/env-builder/roles/generate_namespaces/tasks/main.yaml b/env-builder/roles/generate_namespaces/tasks/main.yaml deleted file mode 100644 index aa2f29ac..00000000 --- a/env-builder/roles/generate_namespaces/tasks/main.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: Set namespace template name from deploy_postfix if it exists - set_fact: - _namespace_template_name: "{{ namespace.deploy_postfix }}" - when: namespace.deploy_postfix is defined and namespace.deploy_postfix - -- name: Set namespace template name from template_path if deploy_postfix not set - set_fact: - _namespace_template_name: "{{ namespace.template_path.split('/') | last | replace('.yml.j2','') | replace('.yaml.j2','') }}" - when: namespace.deploy_postfix is not defined or not namespace.deploy_postfix - -- name: Generate Envs Namespace dir {{ _namespace_template_name }} - ansible.builtin.file: - path: "{{ _current_env_dir }}/Namespaces/{{ _namespace_template_name }}" - state: directory - -- name: Set templates - set_fact: - template_override: "{{ namespace.template_override| default('') }}" - -- name: Generate Envs Namespace yaml {{ _namespace_template_name }} for {{ namespace.name }} - ansible.builtin.blockinfile: - path: "{{ _current_env_dir }}/Namespaces/{{ _namespace_template_name }}/namespace.yml" - block: "{{ lookup('template', namespace.template_path) }}" - create: yes - -- name: Generate Envs Namespace yaml {{ _namespace_template_name }} for {{ namespace.name }} - ansible.builtin.blockinfile: - path: "{{ _current_env_dir }}/Namespaces/{{ _namespace_template_name }}/namespace.yml_override" - block: "{{ template_override }}" - create: yes - when: template_override != '' diff --git a/env-builder/roles/generate_paramsets/tasks/main.yaml b/env-builder/roles/generate_paramsets/tasks/main.yaml deleted file mode 100644 index 1cdc075c..00000000 --- a/env-builder/roles/generate_paramsets/tasks/main.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Get paramset template name from {{ _paramset_path }} - set_fact: - _paramset_template_name: "{{ _paramset_path.split('/')|last|replace('.yml.j2', '')|replace('.yaml.j2', '') }}" - -- name: Get target paramset path for {{ _paramset_path }} - set_fact: - _paramset_target_path: "{{ _paramset_path|replace('.yml.j2', '.yml')|replace('.yaml.j2', '.yml') }}" - -# Try to render the template safely -- name: Try to render paramset {{ _paramset_template_name }} - block: - - name: Render template with template module - ansible.builtin.template: - src: "{{ _paramset_path }}" - dest: "{{ _paramset_target_path }}" - register: render_result - - - name: Log successful paramset generation - debug: - msg: "Successfully generated paramset: {{ _paramset_template_name }}" - rescue: - - name: Log skipped paramset due to missing variables - debug: - msg: "Skipped paramset {{ _paramset_template_name }} - template variables not available in current environment" - - # If the file was created but rendering failed, remove it - - name: Ensure no partial file exists on failure - ansible.builtin.file: - path: "{{ _paramset_target_path }}" - state: absent - ignore_errors: yes - -- name: Delete jinja template {{ _paramset_path}} after rendering - ansible.builtin.file: - state: absent - path: "{{ _paramset_path }}" diff --git a/env-builder/roles/generate_profiles/tasks/main.yaml b/env-builder/roles/generate_profiles/tasks/main.yaml deleted file mode 100644 index 64ce4cf4..00000000 --- a/env-builder/roles/generate_profiles/tasks/main.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Get profile template name from {{ profile.template_path }} - set_fact: - _profile_template_name: "{{ profile.template_path.split('/')|last|replace('.yml.j2', '')|replace('.yaml.j2', '') }}" - -- name: Generate Envs profiles dir - ansible.builtin.file: - path: "{{ _current_env_dir }}/Profiles" - state: directory - -- name: Generate Envs Namespace yaml {{ _profile_template_name }} for {{ profile.name }} - ansible.builtin.blockinfile: - path: "{{ _current_env_dir }}/Profiles/{{ _profile_template_name }}.yml" - block: "{{ lookup('template', profile.template_path) }}" - create: yes diff --git a/env-builder/roles/generate_solution_structure/tasks/main.yaml b/env-builder/roles/generate_solution_structure/tasks/main.yaml deleted file mode 100644 index 1dca18c9..00000000 --- a/env-builder/roles/generate_solution_structure/tasks/main.yaml +++ /dev/null @@ -1,199 +0,0 @@ ---- - -- name: Check if sd file exists in {{ _current_env_dir }}/Inventory/solution-descriptor - stat: - path: "{{ _current_env_dir }}/Inventory/solution-descriptor/sd.{{ item }}" - loop: - - yaml - - yml - register: sd_files_check - ignore_errors: yes - -- name: "Create and fill solution_structure variable" - when: - - sd_files_check is defined - - (sd_files_check.results | selectattr('stat.exists') | list | length) > 0 - block: - - name: Create a path for existing SD file - set_fact: - sd_file_path: "{{ sd_files_check.results | selectattr('stat.exists') | map(attribute='stat.path') | first }}" - - - name: Load applications data from sd.yaml/yml - include_vars: - file: "{{ sd_file_path }}" - name: sd_config - - - name: Check that 'applications'-key exists - assert: - that: - - "'applications' in sd_config" - - "sd_config.applications is sequence" - success_msg: "Root 'applications' key exists and is a list" - fail_msg: "Missing or invalid 'applications' key in root" - - - name: Validate each application entry - assert: - that: - - "'version' in item" - - "'deployPostfix' in item" - - "item.version is string" - - "item.deployPostfix is string" - success_msg: "Valid application entry: {{ item }}" - fail_msg: "Invalid application entry: {{ item | to_nice_json }}" - quiet: true - loop: "{{ sd_config.applications }}" - loop_control: - label: "Application {{ item.version | default('unknown') }}" - - - name: All checks passed - debug: - msg: "All required fields are present and valid!" - - - name: Checking the existence of template descriptor - stat: - path: "{{ templates_dir }}/env_templates/{{ _template_name }}.{{ item }}" - loop: - - yml - - yaml - register: template_descriptor_stat - - - name: Set template descriptor file from found stat - set_fact: - template_descriptor_file: "{{ item.stat.path }}" - loop: "{{ template_descriptor_stat.results }}" - when: item.stat.exists - register: template_descriptor_file_set - - - name: Fail if no template descriptor was found - fail: - msg: "Template descriptor was not found in {{ templates_dir }}/env_templates" - when: template_descriptor_file_set.results | selectattr('skipped', 'defined') | length == (template_descriptor_stat.results | length) - - - name: Read template descriptor file - slurp: - src: "{{ template_descriptor_file }}" - register: env_template_raw - - - name: Parse YAML content in template descriptor - set_fact: - env_template_data: "{{ env_template_raw['content'] | b64decode | from_yaml }}" - - - name: Extract namespace template paths from template descriptor - set_fact: - namespace_template_paths: "{{ env_template_data.namespaces | map(attribute='template_path') | list }}" - when: env_template_data.namespaces is defined - - - name: "Namespace templates: Create temporary template file to render path" - tempfile: - state: file - suffix: .j2 - register: temp_template - - - name: "Namespace templates: Write path to temp file" - copy: - content: "{{ namespace_template_paths }}" - dest: "{{ temp_template.path }}" - - - name: "Namespace templates: Render file and parse JSON" - set_fact: - namespace_template_paths: "{{ lookup('template', temp_template.path) }}" - - - name: "Namespace templates: Render file and parse JSONDelete temporary template file" - file: - path: "{{ temp_template.path }}" - state: absent - - - name: "Namespace templates: list of found namespace templates" - debug: - var: namespace_template_paths - - - name: "Namespace templates: Build template path map by deployPostfix" - set_fact: - postfix_template_map: "{{ dict(postfix_keys | zip(namespace_template_paths)) }}" - vars: - postfix_keys: >- - {{ - namespace_template_paths - | map('basename') - | map('split', '.') - | map('first') - | list - }} - - - name: Initialize solution_structure - no_log: True - set_fact: - solution_structure: {} - - - name: Build solution_structure variable - no_log: True - set_fact: - solution_structure: "{{ solution_structure | combine(_entry | from_json, recursive=True) }}" - loop: "{{ sd_config.applications }}" - loop_control: - label: "{{ item.deployPostfix }}" - vars: - _entry: >- - {% set version_full = item.version -%} - {% set version = version_full.split(':')[1] -%} - {% set solution = version_full.split(':')[0] -%} - {% set postfix = item.deployPostfix -%} - {% set path = postfix_template_map.get(postfix, None) -%} - - {% if path %} - {% set content = lookup('file', path).split('\n') -%} - {% set ns_line = content | select('match', '^\\s*name:') | list | first -%} - {% if ns_line %} - {% set ns_val = ns_line.split(':', 1)[1] | trim | replace('"', '') -%} - {% else %} - {% set ns_val = None -%} - {% endif -%} - {% else %} - {% set ns_val = None -%} - {% endif -%} - - {% set small_dict = { - solution: { - postfix: { - "version": version, - "namespace": ns_val - } - } - } %} - - {{ small_dict | to_json }} - - - name: Update formatting in solution_structure variable - no_log: True - set_fact: - solution_structure: "{{ solution_structure | regex_replace('({{)\\s*([^}\\s]+)\\s*(}})', '\\1 \\2 \\3') }}" - - - name: "Solution_structure: Create temporary template file to render the variable" - tempfile: - state: file - suffix: .j2 - register: temp_template - - - name: "Solution_structure: Write solution_structure to temp template file" - copy: - content: "{{ solution_structure }}" - dest: "{{ temp_template.path }}" - - - name: "Solution_structure: Render template and parse JSON" - no_log: True - set_fact: - solution_structure: "{{ lookup('template', temp_template.path) }}" - - - name: "Solution_structure: Delete temporary template file" - file: - path: "{{ temp_template.path }}" - state: absent - - - name: Show rendered solution_structure - debug: - var: solution_structure - - - name: Merge solution_structure into current_env - no_log: True - set_fact: - current_env: "{{ current_env | combine({'solution_structure': solution_structure}, recursive=True) }}" diff --git a/env-builder/roles/generate_tenant/tasks/main.yaml b/env-builder/roles/generate_tenant/tasks/main.yaml deleted file mode 100644 index 2d6594e2..00000000 --- a/env-builder/roles/generate_tenant/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Generate Tenant yaml {{ _tenant }} - ansible.builtin.blockinfile: - path: "{{ current_env_dir }}/tenant.yml" - block: "{{ lookup('template', _template) }}" - create: yes diff --git a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml index 9c93b558..745fb401 100644 --- a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml +++ b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml @@ -69,9 +69,9 @@ env: DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator" #DOCKER_IMAGE_TAGS - DOCKER_IMAGE_TAG_PIPEGENE: "1.6.9" - DOCKER_IMAGE_TAG_ENVGENE: "1.6.9" - DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.6.9" + DOCKER_IMAGE_TAG_PIPEGENE: "feature_replace_ansible_1part_20251107-104902" + DOCKER_IMAGE_TAG_ENVGENE: "feature_replace_ansible_1part_20251107-104906" + DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "feature_replace_ansible_1part_20251107-105008" jobs: process_environment_variables: diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index ec53a0e9..81ededae 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -1,14 +1,16 @@ -import pathlib import re from os import getenv from .collections_helper import merge_lists -from .yaml_helper import findYamls, openYaml, yaml, writeYamlToFile, store_value_to_yaml, validate_yaml_by_scheme_or_fail +from .yaml_helper import findYamls, openYaml, yaml, writeYamlToFile, store_value_to_yaml, \ + validate_yaml_by_scheme_or_fail from .json_helper import findJsons -from .file_helper import getAbsPath, extractNameFromFile, check_file_exists, check_dir_exists, getParentDirName, extractNameFromDir +from .file_helper import getAbsPath, extractNameFromFile, check_file_exists, check_dir_exists, getParentDirName, \ + extractNameFromDir from .collections_helper import dump_as_yaml_format from .logger import logger from ruyaml.scalarstring import DoubleQuotedScalarString +from pathlib import Path # const INVENTORY_DIR_NAME = "Inventory" @@ -22,39 +24,47 @@ INV_GEN_CREDS_PATH = "Inventory/credentials/inventory_generation_creds.yml" -def find_env_instances_dir(env_name, instances_dir) : +def find_env_instances_dir(env_name, instances_dir): logger.debug(f"Searching for directory {env_name} in {instances_dir}") - dirPointer = pathlib.Path(instances_dir) + dirPointer = Path(instances_dir) dirList = list(dirPointer.rglob(f"{env_name}/Inventory")) logger.debug(f"Search results: {dump_as_yaml_format(dirList)}") if len(dirList) > 1: - logger.error(f"Duplicate directories for {env_name} found in {instances_dir}: \n\t" + ",\n\t".join(str(x) for x in dirList)) - raise ReferenceError(f"Duplicate directories for {env_name} found. Please specify env name with environment folder e.g. sdp-dev/{env_name}.") - for dir in dirList : - if dir.is_dir() : + logger.error(f"Duplicate directories for {env_name} found in {instances_dir}: \n\t" + ",\n\t".join( + str(x) for x in dirList)) + raise ReferenceError( + f"Duplicate directories for {env_name} found. Please specify env name with environment folder e.g. sdp-dev/{env_name}.") + for dir in dirList: + if dir.is_dir(): return str(dir.parent) logger.error(f"Directory for {env_name} is not found in {instances_dir}") raise ReferenceError(f"Can't find directory for {env_name}") + def getenv_and_log(name, *args, **kwargs): var = getenv(name, *args, **kwargs) logger.info(f"{name}: {var}") return var + def getenv_with_error(var_name): var = getenv(var_name) if not var: raise ValueError(f'Required value was not given and is not set in environment as {var_name}') return var + def get_env_instances_dir(environment_name, cluster_name, instances_dir): return f"{instances_dir}/{cluster_name}/{environment_name}" -def check_environment_is_valid_or_fail(environment_name, cluster_name, instances_dir, skip_env_definition_check=False, validate_env_definition_by_schema=False, schemas_dir=""): + +def check_environment_is_valid_or_fail(environment_name, cluster_name, instances_dir, skip_env_definition_check=False, + validate_env_definition_by_schema=False, schemas_dir=""): env_dir = get_env_instances_dir(environment_name, cluster_name, instances_dir) # check that environment directory exists if not check_dir_exists(env_dir): - logger.error(f"Directory for environment '{cluster_name}/{environment_name}' does not exist in: {instances_dir}. We tried to find it in the following path: {env_dir}") + logger.error( + f"Directory for environment '{cluster_name}/{environment_name}' does not exist in: {instances_dir}. We tried to find it in the following path: {env_dir}") raise ReferenceError(f"Validation of environment folder '{env_dir}' failed. See logs above.") # check that env_definition yaml exists env_definition_path = f"{env_dir}/Inventory/env_definition.yml" @@ -63,12 +73,14 @@ def check_environment_is_valid_or_fail(environment_name, cluster_name, instances logger.info(f"Environment {cluster_name}/{environment_name} validation is succesful") return if not check_file_exists(env_definition_path): - logger.error(f"Env_definition.yml is not found in path '{env_definition_path}' for environment {cluster_name}/{environment_name}. Please specify correct env_definition.yml in '{cluster_name}/{environment_name}/Inventory' folder" ) + logger.error( + f"Env_definition.yml is not found in path '{env_definition_path}' for environment {cluster_name}/{environment_name}. Please specify correct env_definition.yml in '{cluster_name}/{environment_name}/Inventory' folder") raise ReferenceError(f"Validation of environment folder '{env_dir}' failed. See logs above.") if validate_env_definition_by_schema: check_env_definition_is_valid_or_fail(env_definition_path, schemas_dir) logger.info(f"Environment {cluster_name}/{environment_name} validation is succesful") + def check_env_definition_is_valid_or_fail(env_definition_path, schemas_dir): schemaPath = f"{schemas_dir}/env-definition.schema.json" if schemas_dir else "schemas/env-definition.schema.json" try: @@ -77,24 +89,31 @@ def check_env_definition_is_valid_or_fail(env_definition_path, schemas_dir): raise ValueError(f"Validation of env_definition in '{env_definition_path} failed. See logs above'") from None -def findResourcesBottomTop(sourceDir, stopParentDir, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern="", searchJsons=False): +def findResourcesBottomTop(sourceDir, stopParentDir, pattern, notPattern="", additionalRegexpPattern="", + additionalRegexpNotPattern="", searchJsons=False): result = [] foundMap = {} # checking that stopParentDir is real parent of sourceDir or we will have infinite loop stopParentDirAbs = getAbsPath(stopParentDir) sourceDirAbs = getAbsPath(sourceDir) - parentPath = pathlib.Path(stopParentDirAbs) - sourcePath = pathlib.Path(sourceDirAbs) + parentPath = Path(stopParentDirAbs) + sourcePath = Path(sourceDirAbs) if parentPath not in sourcePath.parents: logger.error(f"Error while finding resources. {stopParentDirAbs} is not in parents of {sourceDirAbs}.") - raise ReferenceError(f"Error while finding resources. {stopParentDirAbs} is not in parents of {sourceDirAbs}. See logs above.") - return __findResourcesBottomTop__(sourceDir, stopParentDir, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern, searchJsons, result, foundMap) + raise ReferenceError( + f"Error while finding resources. {stopParentDirAbs} is not in parents of {sourceDirAbs}. See logs above.") + return __findResourcesBottomTop__(sourceDir, stopParentDir, pattern, notPattern, additionalRegexpPattern, + additionalRegexpNotPattern, searchJsons, result, foundMap) + -def __findResourcesBottomTop__(sourceDir, stopParentDir, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern, searchJsons, result, foundMap): - logger.debug(f"Searching files in {sourceDir}. Pattern:{pattern}\nNotPattern:{notPattern}\nResult:\n{dump_as_yaml_format(result)}. foundMap:\n{foundMap}") +def __findResourcesBottomTop__(sourceDir, stopParentDir, pattern, notPattern, additionalRegexpPattern, + additionalRegexpNotPattern, searchJsons, result, foundMap): + logger.debug( + f"Searching files in {sourceDir}. Pattern:{pattern}\nNotPattern:{notPattern}\nResult:\n{dump_as_yaml_format(result)}. foundMap:\n{foundMap}") findResults = findYamls(sourceDir, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern) if searchJsons: - findResults = merge_lists(findResults, findJsons(sourceDir, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern)) + findResults = merge_lists(findResults, findJsons(sourceDir, pattern, notPattern, additionalRegexpPattern, + additionalRegexpNotPattern)) for foundFile in findResults: fileName = extractNameFromFile(foundFile) if fileName not in foundMap: @@ -104,14 +123,18 @@ def __findResourcesBottomTop__(sourceDir, stopParentDir, pattern, notPattern, ad result.append(yamlPath) logger.debug(f"Resource added from: {yamlPath}") elif len(findResults) > 1: - logger.error(f"Duplicate resource file with pattern {pattern} found in {sourceDir}: \n\t" + ",\n\t".join(str(x) for x in findResults)) + logger.error( + f"Duplicate resource file with pattern {pattern} found in {sourceDir}: \n\t" + ",\n\t".join( + str(x) for x in findResults)) raise ReferenceError(f"Duplicate resource file with pattern {pattern} found. See logs above.") if getAbsPath(sourceDir) == getAbsPath(stopParentDir): logger.debug(f"Reached parent dir {stopParentDir}. Stopping.") return result else: - parentEnvDirPath = str(pathlib.Path(sourceDir).parent) - return __findResourcesBottomTop__(parentEnvDirPath, stopParentDir, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern, searchJsons, result, foundMap) + parentEnvDirPath = str(Path(sourceDir).parent) + return __findResourcesBottomTop__(parentEnvDirPath, stopParentDir, pattern, notPattern, additionalRegexpPattern, + additionalRegexpNotPattern, searchJsons, result, foundMap) + def getTemplateArtifactName(env_definition_yaml): if "artifact" in env_definition_yaml["envTemplate"]: @@ -121,6 +144,7 @@ def getTemplateArtifactName(env_definition_yaml): gav = env_definition_yaml["envTemplate"]["templateArtifact"]["artifact"] return gav["artifact_id"] + def getEnvDefinition(env_dir): envDefinitionPath = getEnvDefinitionPath(env_dir) if not check_file_exists(envDefinitionPath): @@ -128,9 +152,11 @@ def getEnvDefinition(env_dir): inventoryYaml = openYaml(envDefinitionPath) return inventoryYaml + def getEnvDefinitionPath(env_dir): return f"{env_dir}/{INVENTORY_DIR_NAME}/{ENV_DEFINITION_FILE_NAME}" + def getEnvCredentials(env_dir): envCredentialsPath = getEnvCredentialsPath(env_dir) if not check_file_exists(envCredentialsPath): @@ -139,9 +165,11 @@ def getEnvCredentials(env_dir): credYaml = openYaml(envCredentialsPath) return credYaml + def getEnvCredentialsPath(env_dir): return f"{env_dir}/{CREDENTIALS_DIR_NAME}/{CREDENTIALS_FILE_NAME}" + def getAppDefinitionPath(base_path, template_name): path_without_extension = f"{base_path}/configuration/artifact_definitions/{template_name}" yaml_path = f"{path_without_extension}.yaml" @@ -151,12 +179,15 @@ def getAppDefinitionPath(base_path, template_name): else: return yaml_path + def getTemplateVersionFromEnvDefinition(env_definition_yaml): return env_definition_yaml["generatedVersions"]["generateEnvironmentLatestVersion"] + def getTemplateLatestSnapshotVersion(env_definition_yaml): return env_definition_yaml["env_template_latest_snapshot_version"] + def update_generated_versions(env_dir, stage_tag, template_version=""): comment = "This value is automatically generated during job run." env_definition_yaml = getEnvDefinition(env_dir) @@ -186,18 +217,23 @@ def update_generated_versions(env_dir, stage_tag, template_version=""): writeYamlToFile(env_definition_path, env_definition_yaml) logger.info(f"Generated version {version} updated for stage tag {stage_tag} in environment {env_definition_path}") + def extract_namespace_from_application_path(app_yaml_path): + file_path_posix = Path(app_yaml_path).as_posix() pattern = r'^.+/Namespaces/(.+)/Applications/.+$' - return re.sub(pattern, r'\1', app_yaml_path) + return re.sub(pattern, r'\1', file_path_posix) + def extract_namespace_from_namespace_path(namespace_yaml_path): + file_path_posix = Path(namespace_yaml_path).as_posix() pattern = r'^.+/Namespaces/(.+)/.+$' - return re.sub(pattern, r'\1', namespace_yaml_path) + return re.sub(pattern, r'\1', file_path_posix) def contains_cyrillic(text): return bool(re.search(r'[а-яА-Я]', text)) + def check_for_cyrillic(data, filename, path=''): error_found = False if isinstance(data, dict): @@ -217,17 +253,21 @@ def check_for_cyrillic(data, filename, path=''): if check_for_cyrillic(item, filename, path=f"{path}[{index}]"): error_found = True elif isinstance(item, str) and contains_cyrillic(item): - logger.error(f"Cyrillic character found in list item '{item}' in file '{filename}' at path '{path}[{index}]'") + logger.error( + f"Cyrillic character found in list item '{item}' in file '{filename}' at path '{path}[{index}]'") error_found = True return error_found + def get_cluster_name_from_full_name(env): return env.split('/')[0].strip() + def get_environment_name_from_full_name(env): return env.split('/')[1].strip() -def find_cloud_passport_definition(env_instances_dir, instances_dir) : + +def find_cloud_passport_definition(env_instances_dir, instances_dir): # trying to get explicit passport name from env_definition inventoryYaml = getEnvDefinition(env_instances_dir) cloud_passport_file_name = "" @@ -236,8 +276,9 @@ def find_cloud_passport_definition(env_instances_dir, instances_dir) : if (cloud_passport_file_name): return findPassportByEnvDefinition(env_instances_dir, instances_dir, cloud_passport_file_name) else: - cloudDir = getParentDirName(env_instances_dir+"/") - logger.info(f"Explicit name of cloud passport is not defined. Trying to find by directory in cloud dir {cloudDir}.") + cloudDir = getParentDirName(env_instances_dir + "/") + logger.info( + f"Explicit name of cloud passport is not defined. Trying to find by directory in cloud dir {cloudDir}.") # trying to search passport by cloud name (cloud dir name) cloudDirName = extractNameFromDir(cloudDir) passportFilePath = findPassportInDefaultDirByName(cloudDir, cloudDirName) @@ -246,33 +287,45 @@ def find_cloud_passport_definition(env_instances_dir, instances_dir) : # trying to find passport by default passport name return findPassportInDefaultDirByName(cloudDir, DEFAULT_PASSPORT_NAME) -def findPassportByEnvDefinition(env_instances_dir, instances_dir, cloud_passport_file_name) : - logger.debug(f"Searching for cloud passport file {cloud_passport_file_name} from {env_instances_dir} to {instances_dir}") - passportFiles = findResourcesBottomTop(env_instances_dir, instances_dir, f"/{cloud_passport_file_name}.y", "redentials/") + +def findPassportByEnvDefinition(env_instances_dir, instances_dir, cloud_passport_file_name): + logger.debug( + f"Searching for cloud passport file {cloud_passport_file_name} from {env_instances_dir} to {instances_dir}") + passportFiles = findResourcesBottomTop(env_instances_dir, instances_dir, f"/{cloud_passport_file_name}.y", + "redentials/") if len(passportFiles) == 1: yamlPath = passportFiles[0] logger.info(f"Cloud passport file for {cloud_passport_file_name} found in: {yamlPath}") return yamlPath elif len(passportFiles) > 1: - logger.error(f"Duplicate cloud passport files with key {cloud_passport_file_name} found in {instances_dir}: \n\t" + ",\n\t".join(str(x) for x in passportFiles)) - raise ReferenceError(f"Duplicate cloud passport files with key {cloud_passport_file_name} found. See logs above.") + logger.error( + f"Duplicate cloud passport files with key {cloud_passport_file_name} found in {instances_dir}: \n\t" + ",\n\t".join( + str(x) for x in passportFiles)) + raise ReferenceError( + f"Duplicate cloud passport files with key {cloud_passport_file_name} found. See logs above.") else: raise ReferenceError(f"Cloud passport with key {cloud_passport_file_name} not found in {instances_dir}") -def findPassportInDefaultDirByName(env_instances_dir, passport_name) : - logger.debug(f"Searching for yaml with pattern '{DEFAULT_PASSPORT_DIR_NAME}/{passport_name}.y' in {env_instances_dir}") - passportFiles = findYamls(env_instances_dir, f"{DEFAULT_PASSPORT_DIR_NAME}/{passport_name}.y", notPattern="redentials/") + +def findPassportInDefaultDirByName(env_instances_dir, passport_name): + logger.debug( + f"Searching for yaml with pattern '{DEFAULT_PASSPORT_DIR_NAME}/{passport_name}.y' in {env_instances_dir}") + passportFiles = findYamls(env_instances_dir, f"{DEFAULT_PASSPORT_DIR_NAME}/{passport_name}.y", + notPattern="redentials/") if len(passportFiles) == 1: yamlPath = passportFiles[0] logger.info(f"Cloud passport file for {env_instances_dir} with name {passport_name} found in: {yamlPath}") return yamlPath elif len(passportFiles) > 1: - logger.error(f"More than one passport files found for env {env_instances_dir} with name {passport_name}:\n{dump_as_yaml_format(passportFiles)}") - raise ReferenceError(f"More than one passport files found for env {env_instances_dir} with name {passport_name}. See logs above.") + logger.error( + f"More than one passport files found for env {env_instances_dir} with name {passport_name}:\n{dump_as_yaml_format(passportFiles)}") + raise ReferenceError( + f"More than one passport files found for env {env_instances_dir} with name {passport_name}. See logs above.") else: logger.info(f"Cloud passport for env {env_instances_dir} with name {passport_name} is not found. ") return "" + def find_cloud_name_from_passport(source_env_dir, all_instances_dir): # checking if inventory is related to cloud passport inventoryYaml = getEnvDefinition(source_env_dir) @@ -293,4 +346,3 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): return cloudPassportFileName else: return "" - diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index 8ebd1765..056dcf1b 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -1,24 +1,30 @@ -import pathlib import os import glob import re import shutil from typing import Callable +from pathlib import Path + from .logger import logger + def extractNameFromFile(filePath): - return pathlib.Path(filePath).stem + return Path(filePath).stem + def extractNameWithExtensionFromFile(filePath): - return pathlib.Path(filePath).name + return Path(filePath).name + def extractNameFromDir(dirName): - return pathlib.Path(dirName).stem + return Path(dirName).stem + -def check_dir_exists(dir_path) : - dir = pathlib.Path(dir_path) +def check_dir_exists(dir_path): + dir = Path(dir_path) return dir.exists() and dir.is_dir() + def identify_yaml_extension(file_path: str) -> str: """ Takes file_path and check if it exists either with .yml or .yaml extension and returns existing file @@ -30,60 +36,94 @@ def identify_yaml_extension(file_path: str) -> str: return file raise FileNotFoundError(f"Neither of these files: {possible_files} exist.") + def find_all_sub_dir(dir_path): return os.walk(dir_path) -def check_file_exists(file_path) : - file = pathlib.Path(file_path) + +def check_file_exists(file_path): + file = Path(file_path) return file.exists() and file.is_file() -def check_dir_exist_and_create(dir_path) : + +def check_dir_exist_and_create(dir_path): logger.debug(f'Checking that dir exists or create dir in path: {dir_path}') os.makedirs(dir_path, exist_ok=True) -def delete_dir(path) : + +def delete_dir(path): try: shutil.rmtree(path) except: logger.info(f'{path} directory does not exist') -def copy_path(source_path, target_path) : - # check that we are not trying to copy file to itself, or 'cp' will exit with error - if getDirName(source_path) == getDirName(target_path) and (check_file_exists(source_path) and source_path == target_path + extractNameWithExtensionFromFile(source_path)): - logger.info(f"Trying to copy {source_path} to itself (target path: {target_path}). Skipping...") - elif glob.glob(source_path) : - logger.info(f'Copying from {source_path} to {target_path}') - logger.debug(f'Checking target path {target_path} exists: {os.path.exists(target_path)}') - if not os.path.exists(target_path) : - if os.path.isdir(target_path) : - dirPath = target_path - else : - dirPath = os.path.dirname(target_path) - logger.debug(f'Creating dir {dirPath}') - os.makedirs(dirPath, exist_ok=True) - exit_code = os.system(f"cp -rf {source_path} {target_path}") - if (exit_code) : - logger.error(f"Error during copying from {source_path} to {target_path}") - exit(1) - else : - logger.info(f"Path {source_path} doesn't exist. Skipping...") -def move_path(source_path, target_path) : - if glob.glob(source_path) : +def is_glob(path: str) -> bool: + return any(ch in str(path) for ch in ["*", "?", "["]) + + +def is_source_path_valid(source_path: Path, target_path: Path) -> bool: + if not source_path.exists(): + logger.info(f"Path {source_path} doesn't exist. Skipping...") + return False + if source_path == target_path: + logger.info(f"Trying to copy {source_path} to itself ({target_path}). Skipping...") + return False + return True + + +def _is_glob(path: str) -> bool: + return any(ch in str(path) for ch in ["*", "?", "["]) + + +def copy_path(source_path: str, target_dir: str): + target_dir = Path(target_dir) + target_dir.mkdir(parents=True, exist_ok=True) + + if _is_glob(source_path): + matches = sorted(glob.glob(source_path)) + if not matches: + return + for match in matches: + _copy_single(Path(match), target_dir, is_from_glob=True) + else: + _copy_single(Path(source_path), target_dir, is_from_glob=False) + + +def _copy_single(src: Path, target_dir: Path, is_from_glob: bool): + is_source_path_valid(src, target_dir) + + if src.is_dir(): + for item in src.rglob("*"): + rel = item.relative_to(src.parent if is_from_glob else src) + dst = target_dir / rel + if item.is_dir(): + dst.mkdir(parents=True, exist_ok=True) + else: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, dst) + elif src.is_file(): + dst = target_dir / src.name + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + +def move_path(source_path, target_path): + if glob.glob(source_path): logger.info(f'Moving from {source_path} to {target_path}') logger.debug(f'Checking target path {target_path} exists: {os.path.exists(target_path)}') - if not os.path.exists(target_path) : - if os.path.isdir(target_path) : + if not os.path.exists(target_path): + if os.path.isdir(target_path): dirPath = target_path - else : + else: dirPath = os.path.dirname(target_path) logger.debug(f'Creating dir {dirPath}') os.makedirs(dirPath, exist_ok=True) exit_code = os.system(f"mv -f {source_path} {target_path}") - if (exit_code) : + if (exit_code): logger.error(f"Error during Moving from {source_path} to {target_path}") exit(1) - else : + else: logger.info(f"Path {source_path} doesn't exist. Skipping...") @@ -92,33 +132,41 @@ def openFileAsString(filePath): result = f.read() return result + def deleteFile(filePath): os.remove(filePath) + def writeToFile(filePath, contents): os.makedirs(os.path.dirname(filePath), exist_ok=True) with open(filePath, 'w+') as f: f.write(contents) return + def getAbsPath(path): return os.path.abspath(path) + def getRelPath(path, start_path=None): if start_path: return os.path.relpath(path, start_path) return os.path.relpath(path, os.getenv('CI_PROJECT_DIR')) + def get_parent_dir_for_dir(dirPath): - path = pathlib.Path(dirPath) + path = Path(dirPath) return str(path.parent.absolute()) + def getDirName(filePath): return os.path.dirname(filePath) + def getParentDirName(filePath): return os.path.dirname(getDirName(filePath)) + def get_files_with_filter(path_to_filter: str, filter: Callable[[str], bool]) -> set[str]: matching_files = set() for root, _, files in os.walk(path_to_filter): @@ -128,50 +176,50 @@ def get_files_with_filter(path_to_filter: str, filter: Callable[[str], bool]) -> matching_files.add(filepath) return matching_files + def findAllFilesInDir(dir, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern=""): result = [] - dirPointer = pathlib.Path(dir) + dirPointer = Path(dir) fileList = list(dirPointer.rglob("*.*")) for f in fileList: result.append(str(f)) return findFiles(result, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern) -def findFiles(fileList, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern="") : + +def findFiles(fileList: list[Path], pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern=""): result = [] for filePath in fileList: + # this ensures that pattern matching works correctly on both Windows (\) and Unix (/) + file_path_posix = Path(filePath).as_posix() if ( - pattern in filePath - and (notPattern=="" or notPattern not in filePath) - and (additionalRegexpPattern=="" or re.match(additionalRegexpPattern, filePath)) - and (additionalRegexpNotPattern=="" or not re.match(additionalRegexpNotPattern, filePath)) + pattern in file_path_posix + and (notPattern == "" or notPattern not in file_path_posix) + and (additionalRegexpPattern == "" or re.match(additionalRegexpPattern, file_path_posix)) + and (additionalRegexpNotPattern == "" or not re.match(additionalRegexpNotPattern, file_path_posix)) ): result.append(filePath) - logger.debug(f"Path {filePath} match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") + logger.debug( + f"Path {filePath} match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") else: - logger.debug(f"Path {filePath} doesn't match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") + logger.debug( + f"Path {filePath} doesn't match pattern: {pattern} or notPattern: {notPattern} or additionalPattern: {additionalRegexpPattern}") return result -def removeAnsibleTrashFromFile(filePath) : - ansible_trash = [ - "# BEGIN ANSIBLE MANAGED BLOCK\n---", - "# END ANSIBLE MANAGED BLOCK", - "# BEGIN ANSIBLE MANAGED BLOCK" - ] - with open(filePath, 'r') as f: - fileContent = f.read() - for trash in ansible_trash: - fileContent = fileContent.replace(trash, "") - with open(filePath, 'w') as f: - f.write(fileContent) - -def get_all_files_in_dir(dir, pathToRemove=""): +def get_all_files_in_dir(dir): + dir_path = Path(dir) result = [] - dirPath = pathlib.Path(dir) - for item in dirPath.rglob("*"): + for item in dir_path.rglob("*"): if item.is_file(): - itemStr = str(item) - if pathToRemove: - itemStr = itemStr.replace(pathToRemove, "") - result.append(itemStr) + result.append(str(item.relative_to(dir_path))) return result + + +def ensure_directory(path: Path, mode: int): + if not path.exists(): + path.mkdir(parents=True, exist_ok=True) + logger.info(f"Created directory: {path}") + else: + logger.info(f"Directory already exists: {path}") + path.chmod(mode) + diff --git a/python/envgene/envgenehelper/validation.py b/python/envgene/envgenehelper/validation.py new file mode 100644 index 00000000..a5a6aeec --- /dev/null +++ b/python/envgene/envgenehelper/validation.py @@ -0,0 +1,21 @@ +def ensure_required_keys(context: dict, required: list[str]): + missing = [var for var in required if var not in context] + if missing: + raise ValueError( + f"Required variables: {', '.join(required)}. " + f"Not found: {', '.join(missing)}" + ) + + +def ensure_valid_fields(context: dict, fields: list[str]): + invalid = [] + for field in fields: + value = context.get(field) + if not value: + invalid.append(f"{field}={value!r}") + + if invalid: + raise ValueError( + f"Invalid or empty fields found: {', '.join(invalid)}. " + f"Required fields: {', '.join(fields)}" + ) diff --git a/python/envgene/envgenehelper/yaml_helper.py b/python/envgene/envgenehelper/yaml_helper.py index 58893ecd..9ba15e45 100644 --- a/python/envgene/envgenehelper/yaml_helper.py +++ b/python/envgene/envgenehelper/yaml_helper.py @@ -14,9 +14,11 @@ from ruyaml import CommentedMap, CommentedSeq from typing import Callable, OrderedDict + def create_yaml_processor(is_safe=False) -> ruyaml.main.YAML: def _null_representer(self: ruyaml.representer.BaseRepresenter, data: None) -> ruyaml.Any: return self.represent_scalar('tag:yaml.org,2002:null', 'null') + if is_safe: yaml = ruyaml.main.YAML(typ='safe') else: @@ -27,10 +29,12 @@ def _null_representer(self: ruyaml.representer.BaseRepresenter, data: None) -> r yaml.Representer.add_representer(type(None), _null_representer) return yaml + def get_empty_yaml(): return ruyaml.CommentedMap() -def openYaml(filePath, safe_load=False, default_yaml: Callable=get_empty_yaml, allow_default=False): + +def openYaml(filePath, safe_load=False, default_yaml: Callable = get_empty_yaml, allow_default=False): if allow_default and not check_file_exists(filePath): logger.info(f'{filePath} not found. Returning default value') return default_yaml() @@ -40,6 +44,7 @@ def openYaml(filePath, safe_load=False, default_yaml: Callable=get_empty_yaml, a resultYaml = readYaml(f.read(), safe_load, context=f"File: {filePath}") return resultYaml + def readYaml(text, safe_load=False, context=None): if text is None: resultYaml = None @@ -53,11 +58,13 @@ def readYaml(text, safe_load=False, context=None): return get_empty_yaml() return resultYaml + def convert_dict_to_yaml(d): if isinstance(d, ruyaml.CommentedMap): return d return ruyaml.CommentedMap(d) + def remove_empty_list_comments(data): # There are cases when list has values and those values have comments related # to them. When all values are removed from list, comments are left behind @@ -77,20 +84,24 @@ def remove_empty_list_comments(data): if isinstance(item, (CommentedMap, CommentedSeq)): remove_empty_list_comments(item) + def writeYamlToFile(filePath, contents): - logger.debug(f"Writing yaml to file: {filePath}") + filePath = Path(filePath) + logger.info(f"Writing yaml to file: {filePath}") os.makedirs(os.path.dirname(filePath), exist_ok=True) remove_empty_list_comments(contents) with open(filePath, 'w+') as f: yaml.dump(contents, f) return + def dumpYamlToStr(content): buffer = StringIO() yaml.dump(content, buffer) return buffer.getvalue() -def addHeaderToYaml(file_path: str, header_text: str) : + +def addHeaderToYaml(file_path: str, header_text: str): if (header_text): logger.debug(f'Adding header {header_text} to yaml: {file_path}') file_contents = openFileAsString(file_path) @@ -98,16 +109,19 @@ def addHeaderToYaml(file_path: str, header_text: str) : comment_text = "# " + header_text.replace("\n", "\n# ") writeToFile(file_path, comment_text + "\n" + file_contents) -def alignYamlFileComments(file_path) : + +def alignYamlFileComments(file_path): logger.debug(f'Alligning comments yaml: {file_path}') yamlData = openYaml(file_path) alignYamlComments(yamlData, 0) writeYamlToFile(file_path, yamlData) + def deleteCommentByKey(yamlContent, key): if yamlContent.ca: yamlContent.ca.items.pop(key, None) + def alignYamlComments(yamlContent, extra_indent=0): is_dict = isinstance(yamlContent, dict) if not is_dict and not isinstance(yamlContent, list): @@ -117,12 +131,13 @@ def alignYamlComments(yamlContent, extra_indent=0): if comments: max_col = max(map(lambda x: x[2].column if x[2] else 0, comments), default=0) for comment in comments: - if (comment[2]) : + if (comment[2]): comment[2].column = max_col + extra_indent for element in (yamlContent.values() if is_dict else yamlContent): alignYamlComments(element, extra_indent=extra_indent) return None + def sortYaml(yaml_data, schema_path, remove_additional_props): with open(schema_path, 'r') as f: schema_data = json.load(f) @@ -136,6 +151,7 @@ def sortYaml(yaml_data, schema_path, remove_additional_props): ) return sort_data + def get_nested_yaml_attribute_or_fail(yaml_content, attribute_str): keys = attribute_str.split('.') sub_content = yaml_content @@ -147,6 +163,7 @@ def get_nested_yaml_attribute_or_fail(yaml_content, attribute_str): sub_content = sub_content[key] return sub_content + def ensure_nested_attr_parents_exist(yaml_content, attribute_str): keys = attribute_str.split('.') sub_content = yaml_content @@ -157,16 +174,19 @@ def ensure_nested_attr_parents_exist(yaml_content, attribute_str): last_key = keys[-1] return sub_content, last_key + def ensure_nested_attr_exists(yaml_content, attribute_str, default_value=None): sub_content, last_key = ensure_nested_attr_parents_exist(yaml_content, attribute_str) if not last_key in sub_content: sub_content[last_key] = default_value return sub_content, last_key + def get_or_create_nested_yaml_attribute(yaml_content, attribute_str, default_value=None): yaml_content, key = ensure_nested_attr_exists(yaml_content, attribute_str, default_value) return yaml_content[key] + def set_nested_yaml_attribute(yaml_content, attribute_str, value, comment="", is_overwriting=True): if value is None: return @@ -181,9 +201,12 @@ def set_nested_yaml_attribute(yaml_content, attribute_str, value, comment="", is else: logger.debug(f'Setting {attribute_str} is skipped. Attribute already exists and is_overwriting is set to False') + primitiveTypes = (int, str, bool, float) -def merge_yaml_into_target(yaml_content, target_attribute_str, source_yaml, overwrite_existing_values=True, overwrite_existing_comments=True): + +def merge_yaml_into_target(yaml_content, target_attribute_str, source_yaml, overwrite_existing_values=True, + overwrite_existing_comments=True): if source_yaml is None: return source_yaml = convert_dict_to_yaml(source_yaml) @@ -206,11 +229,12 @@ def merge_yaml_into_target(yaml_content, target_attribute_str, source_yaml, over if isinstance(target_yaml[k], dict) and isinstance(v, dict): merge_yaml_into_target(target_yaml[k], "", v, overwrite_existing_values, overwrite_existing_comments) elif isinstance(target_yaml[k], list) and isinstance(v, list): - target_yaml[k].extend(v_el for v_el in v if v_el not in target_yaml[k] and (isinstance(v_el, primitiveTypes) or isinstance(v_el, list))) + target_yaml[k].extend(v_el for v_el in v if v_el not in target_yaml[k] and ( + isinstance(v_el, primitiveTypes) or isinstance(v_el, list))) src_dicts = {} for v_k, v_el in enumerate(v): if isinstance(v_el, dict): - src_dicts.update({v_k:v_el}) + src_dicts.update({v_k: v_el}) for t_k, t_el in enumerate(target_yaml[k]): if not isinstance(t_el, dict): continue @@ -222,13 +246,15 @@ def merge_yaml_into_target(yaml_content, target_attribute_str, source_yaml, over merge = True break if merge: - target_yaml[k][t_k] = merge_yaml_into_target(t_el, '', src_dicts[t_k], overwrite_existing_values, overwrite_existing_comments) + target_yaml[k][t_k] = merge_yaml_into_target(t_el, '', src_dicts[t_k], overwrite_existing_values, + overwrite_existing_comments) del src_dicts[k] for _, src_dicts_el in src_dicts.items(): target_yaml[k].append(src_dicts_el) elif overwrite_existing_values: target_yaml[k] = v + def store_value_to_yaml(yamlContent, key, value, comment=""): logger.debug(f"Updating key {key} with value {value} in yaml") if key in yamlContent: @@ -240,14 +266,16 @@ def store_value_to_yaml(yamlContent, key, value, comment=""): else: yamlContent.insert(1, key, value) + def merge_dict_key_with_comment(targetKey, targetYaml, sourceKey, sourceYaml, comment=""): if sourceKey in sourceYaml: - deleteCommentByKey(targetYaml, targetKey) - deleteCommentByKey(sourceYaml, sourceKey) - store_value_to_yaml(targetYaml, targetKey, sourceYaml[sourceKey], comment) + deleteCommentByKey(targetYaml, targetKey) + deleteCommentByKey(sourceYaml, sourceKey) + store_value_to_yaml(targetYaml, targetKey, sourceYaml[sourceKey], comment) -def beautifyYaml(file_path, schema_path="", header_text="", allign_comments=False, wrap_all_strings=False, remove_additional_props = False): +def beautifyYaml(file_path, schema_path="", header_text="", allign_comments=False, wrap_all_strings=False, + remove_additional_props=False): logger.info(f'Beautifying yaml: {file_path} with schema: {schema_path}') yamlData = openYaml(file_path) if schema_path: @@ -263,29 +291,33 @@ def beautifyYaml(file_path, schema_path="", header_text="", allign_comments=Fals if allign_comments: alignYamlFileComments(file_path) -def findYamls(dir, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern="") : + +def findYamls(dir, pattern, notPattern="", additionalRegexpPattern="", additionalRegexpNotPattern=""): fileList = findAllYamlsInDir(dir) return findFiles(fileList, pattern, notPattern, additionalRegexpPattern, additionalRegexpNotPattern) -def findAllYamlsInDir(dir) : + +def findAllYamlsInDir(dir): result = [] dirPointer = pathlib.Path(dir) fileList = list(dirPointer.rglob("*.yml")) fileListYaml = list(dirPointer.rglob("*.yaml")) - if len(fileListYaml) > 0 : + if len(fileListYaml) > 0: fileList = fileList + fileListYaml - for f in fileList : + for f in fileList: result.append(str(f)) return result -def mergeYamlInDir(dir_path) : + +def mergeYamlInDir(dir_path): result = {} if check_dir_exists(dir_path): yamlList = findAllYamlsInDir(dir_path) - for yamlPath in yamlList : + for yamlPath in yamlList: result.update(openYaml(yamlPath)) return result + def make_quotes_for_all_strings(yaml_data): if isinstance(yaml_data, (dict, list, set)): for k, v in (yaml_data.items() if isinstance(yaml_data, dict) else enumerate(yaml_data)): @@ -293,6 +325,7 @@ def make_quotes_for_all_strings(yaml_data): yaml_data[k] = DoubleQuotedScalarString(v) make_quotes_for_all_strings(v) + def make_quotes_for_strings(yaml_data): if isinstance(yaml_data, (dict, list, set)): for k, v in (yaml_data.items() if isinstance(yaml_data, dict) else enumerate(yaml_data)): @@ -303,6 +336,7 @@ def make_quotes_for_strings(yaml_data): else: make_quotes_for_strings(v) + def align_spaces_before_comments(filePath): result = "" f = open(filePath, 'r') @@ -317,6 +351,7 @@ def align_spaces_before_comments(filePath): writeToFile(filePath, result) + def copy_yaml_and_remove_empty_dicts(source_yaml): # copying yaml dict result = copy.deepcopy(source_yaml) @@ -326,19 +361,22 @@ def copy_yaml_and_remove_empty_dicts(source_yaml): if isinstance(value, dict): result[key] = copy_yaml_and_remove_empty_dicts(value) # removing all empty keys - empty_keys = [k for k,v in result.items() if v == {}] + empty_keys = [k for k, v in result.items() if v == {}] for k in empty_keys: del result[k] return result + def empty_yaml(): result = yaml.load("{}") return result + def yaml_from_string(yaml_str): result = yaml.load(yaml_str) return result + def validate_yaml_by_scheme_or_fail(yaml_file_path: str, schema_file_path: str) -> None: yaml_content = openYaml(yaml_file_path) schema_content = openJson(schema_file_path) @@ -350,6 +388,7 @@ def validate_yaml_by_scheme_or_fail(yaml_file_path: str, schema_file_path: str) log_jsonschema_validation_error(err) raise ValueError(f"Validation failed") from None + def validate_yaml_data_by_scheme(data, schema, cls=None, *args, **kwargs): if cls is None: cls = jsonschema.validators.validator_for(schema) @@ -358,6 +397,7 @@ def validate_yaml_data_by_scheme(data, schema, cls=None, *args, **kwargs): errors = sorted(validator.iter_errors(data), key=lambda e: e.path) return errors + def log_jsonschema_validation_error(err): key_path = '.'.join(str(index) for index in err.absolute_path) if isinstance(err.instance, OrderedDict): @@ -372,6 +412,7 @@ def log_jsonschema_validation_error(err): logger.error(err.message) logger.error(f"\n") + def convert_ordereddict_to_dict(obj): if isinstance(obj, OrderedDict): return {k: convert_ordereddict_to_dict(v) for k, v in obj.items()} @@ -380,6 +421,16 @@ def convert_ordereddict_to_dict(obj): else: return obj + +def find_all_yaml_files_by_stem(path: str): + file_paths = [] + for ext in ["yaml", "yml"]: + file_path = Path(f"{path}.{ext}") + if file_path.exists(): + file_paths.append(file_path) + return file_paths + + jschon.create_catalog('2020-12') yaml = create_yaml_processor() safe_yaml = create_yaml_processor(is_safe=True) diff --git a/scripts/build_env/build_env.py b/scripts/build_env/build_env.py index 6b8df224..9caae2f1 100644 --- a/scripts/build_env/build_env.py +++ b/scripts/build_env/build_env.py @@ -9,19 +9,23 @@ from resource_profiles import processResourceProfiles from schema_validation import checkEnvSpecificParametersBySchema from cloud_passport import process_cloud_passport +from pathlib import Path # const GENERATED_HEADER = "The contents of this file is generated from template artifact: %s.\nContents will be overwritten by next generation.\nPlease modify this contents only for development purposes or as workaround." -def findNamespaces(dir) : + +def find_namespaces(dir): result = [] fileList = findAllYamlsInDir(dir) for filePath in fileList: - if "/Namespaces/" in filePath: + path = Path(filePath) + if "Namespaces" in path.parts: result.append(filePath) logger.info(f'List of {dir} namespaces: \n {dump_as_yaml_format(result)}') return result + def processFileList(mask, dict, dirPointer): fileList = list(dirPointer.rglob(mask)) for f in fileList: @@ -34,6 +38,7 @@ def processFileList(mask, dict, dirPointer): dict[extractNameFromFile(filePath)] = [{"filePath": filePath, "envSpecific": False}] return dict + def createParamsetsMap(dir): result = {} dirPointer = pathlib.Path(dir) @@ -43,14 +48,16 @@ def createParamsetsMap(dir): logger.debug(f'List of {dir} paramsets: \n %s', dump_as_yaml_format(result)) return result -def sortParameters(params) : + +def sortParameters(params): result = copy.deepcopy(params) result.clear() for k in sorted(params.keys()): - result[k] = params[k] + result[k] = params[k] return result -def openParamset(path, template_context=None) : + +def openParamset(path, template_context=None): if path.endswith(".json"): return openJson(path) # using safe load to load without comments @@ -64,13 +71,15 @@ def openParamset(path, template_context=None) : paramsetYaml = openYaml(path, safe_load=True) return paramsetYaml -def findParamsetsInDir(dirPath) : + +def findParamsetsInDir(dirPath): fileList = findAllYamlsInDir(dirPath) fileListJson = findAllJsonsInDir(dirPath) - if len(fileListJson) > 0 : + if len(fileListJson) > 0: return fileList + fileListJson return fileList + def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): # Walk up the directory tree until we find the Inventory/env_definition.yml file current_dir = os.path.dirname(templatePath) @@ -94,10 +103,12 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): # Validate derived names if not derived_cluster_name or not derived_env_name: logger.error(f"Invalid folder structure: empty cluster or environment name in path {current_dir}") - raise ReferenceError(f"Invalid folder structure for environment derivation. Expected: environments///, found: {current_dir}. Please check the folder structure.") + raise ReferenceError( + f"Invalid folder structure for environment derivation. Expected: environments///, found: {current_dir}. Please check the folder structure.") elif not re.match(r'^[a-zA-Z0-9_-]+$', derived_env_name): # Keep invalid name as warning only, don't fail - logger.warning(f"Invalid environment name '{derived_env_name}' derived from path {current_dir}. Only alphanumeric characters, hyphens, and underscores are allowed.") + logger.warning( + f"Invalid environment name '{derived_env_name}' derived from path {current_dir}. Only alphanumeric characters, hyphens, and underscores are allowed.") # Continue with the invalid name, downstream validation will handle it valid_structure_found = True else: @@ -105,13 +116,15 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): else: # Missing cluster or environment level in hierarchy logger.error(f"Invalid folder structure: missing required hierarchy levels in path {current_dir}") - raise ReferenceError(f"Invalid folder structure. Expected: environments///, found incomplete hierarchy in: {current_dir}. Please check the folder structure.") + raise ReferenceError( + f"Invalid folder structure. Expected: environments///, found incomplete hierarchy in: {current_dir}. Please check the folder structure.") if os.path.exists(env_def_path): env_definition = openYaml(env_def_path) # If environmentName is not defined, use the derived name - if "inventory" in env_definition and not env_definition["inventory"].get("environmentName") and derived_env_name: + if "inventory" in env_definition and not env_definition["inventory"].get( + "environmentName") and derived_env_name: logger.info("Deriving environment name '" + derived_env_name + "' from folder structure") env_definition["inventory"]["environmentName"] = derived_env_name @@ -132,7 +145,8 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): env_definition = openYaml(source_path) # If environmentName is not defined, use the derived name - if "inventory" in env_definition and not env_definition["inventory"].get("environmentName") and derived_env_name: + if "inventory" in env_definition and not env_definition["inventory"].get( + "environmentName") and derived_env_name: logger.info("Deriving environment name '" + derived_env_name + "' from folder structure") env_definition["inventory"]["environmentName"] = derived_env_name @@ -146,12 +160,15 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): # Strict validation: If we reach here, no valid folder structure was found if not valid_structure_found: - logger.error(f"Invalid folder structure: Could not determine environment name from path for template {templatePath}") - raise ReferenceError(f"Invalid folder structure. Expected: environments///Inventory/env_definition.yml, but could not find valid hierarchy in path for template {templatePath}. Please check the folder structure.") + logger.error( + f"Invalid folder structure: Could not determine environment name from path for template {templatePath}") + raise ReferenceError( + f"Invalid folder structure. Expected: environments///Inventory/env_definition.yml, but could not find valid hierarchy in path for template {templatePath}. Please check the folder structure.") # If we have valid structure but no env_definition.yml file found if derived_env_name and valid_structure_found: - logger.warning(f"Valid folder structure found but env_definition.yml missing. Creating minimal environment definition with derived name '{derived_env_name}'") + logger.warning( + f"Valid folder structure found but env_definition.yml missing. Creating minimal environment definition with derived name '{derived_env_name}'") return { "inventory": { "environmentName": derived_env_name @@ -162,15 +179,30 @@ def findEnvDefinitionFromTemplatePath(templatePath, env_instances_dir=None): raise ReferenceError(f"Environment definition not found for template {templatePath}") -def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, parametersTag, paramset_map, env_specific_params_map, header_text="", env_instances_dir=None): +def sort_paramsets_with_same_name(entries: list[dict]) -> list[dict]: + # strict order processing paramsets template -> instance + def sort_key(e): + path = e["filePath"] + if "from_template" in path: + return 1, path + elif "from_instance" in path: + return 2, path + return 0, path + + return sorted(entries, key=sort_key) + + +def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, parametersTag, paramset_map, + env_specific_params_map, header_text="", env_instances_dir=None): params = copy.deepcopy(paramsTemplate[parametersTag]) for pset in paramsTemplate[paramsetsTag]: # Check if paramset exists in paramset_map before accessing it if pset not in paramset_map: - logger.warning(f"Paramset '{pset}' referenced in {paramsetsTag} for template '{templatePath}' was not found. It may have been skipped due to missing variables.") + logger.warning( + f"Paramset '{pset}' referenced in {paramsetsTag} for template '{templatePath}' was not found. It may have been skipped due to missing variables.") continue - paramSetDefinition = paramset_map[pset] + paramSetDefinition = sort_paramsets_with_same_name(paramset_map[pset]) for entry in paramSetDefinition: paramSetFile = entry["filePath"] logger.info(f"Processing paramset {pset} in file {paramSetFile}") @@ -195,14 +227,20 @@ def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, "cloud": cloud_name, "cloudNameWithCluster": cloud_name_with_cluster, "solution_structure": env_definition.get("solutionStructure", {}), - "additionalTemplateVariables": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", {}), + "additionalTemplateVariables": env_definition.get("envTemplate", {}).get( + "additionalTemplateVariables", {}), "cluster": { "name": cluster_name, # Add cluster-specific properties that might be used in templates - "cloud_api_url": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", {}).get("cloud_api_url", ""), - "cloud_api_port": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", {}).get("cloud_api_port", ""), - "cloud_public_url": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", {}).get("cloud_public_url", ""), - "cloud_api_protocol": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", {}).get("cloud_api_protocol", "https") + "cloud_api_url": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", + {}).get("cloud_api_url", ""), + "cloud_api_port": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", + {}).get("cloud_api_port", ""), + "cloud_public_url": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", + {}).get("cloud_public_url", ""), + "cloud_api_protocol": env_definition.get("envTemplate", {}).get("additionalTemplateVariables", + {}).get("cloud_api_protocol", + "https") } } template_context = { @@ -217,15 +255,16 @@ def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, # paramSetName = paramSetValues["name"] paramSetVersion = paramSetValues["version"] if "version" in paramSetValues else "n/a" - if "parameters" in paramSetValues : + if "parameters" in paramSetValues: paramSetParameters = paramSetValues["parameters"] - else : + else: paramSetParameters = {} - if "applications" in paramSetValues : + if "applications" in paramSetValues: paramSetAppParams = list(paramSetValues["applications"]) - else : + else: paramSetAppParams = [] - paramsetDefinitionComment = "paramset: " + paramSetName + " version: " + str(paramSetVersion) + " source: " + ("template" if "from_template" in paramSetFile else "instance") + paramsetDefinitionComment = "paramset: " + paramSetName + " version: " + str( + paramSetVersion) + " source: " + ("template" if "from_template" in paramSetFile else "instance") # process parameters in ParamSet for k in paramSetParameters: # get value with potential merge of dicts @@ -235,27 +274,32 @@ def convertParameterSetsToParameters(templatePath, paramsTemplate, paramsetsTag, if isEnvSpecificParamset: storeToEnvSpecificParametersMap(env_specific_params_map, "", parametersTag, k, val, pset) # prepare application parameters - convertParameterSetsToApplication(templatePath, paramsetDefinitionComment, paramSetAppParams, pset, parametersTag, isEnvSpecificParamset, env_specific_params_map, header_text) + convertParameterSetsToApplication(templatePath, paramsetDefinitionComment, paramSetAppParams, pset, + parametersTag, isEnvSpecificParamset, env_specific_params_map, + header_text) params = sortParameters(params) return params -def convertParameterSetsToApplication(templatePath, paramsetDefinitionComment, applicationsParamSets, paramsetName, parametersTag, isEnvSpecificParamset, env_specific_params_map, header_text=""): - application_schema="schemas/application.schema.json" + +def convertParameterSetsToApplication(templatePath, paramsetDefinitionComment, applicationsParamSets, paramsetName, + parametersTag, isEnvSpecificParamset, env_specific_params_map, header_text=""): + application_schema = "schemas/application.schema.json" for appParams in applicationsParamSets: - appName = appParams["appName"] if "appName" in appParams else appParams["name"] - applicationParametersFile = os.path.dirname(templatePath) + "/Applications/" + appName + ".yml" - appDefinition = getApplicationParametersYaml(appName, applicationParametersFile) - for j in appParams["parameters"]: - # get value with potential merge of dicts - val = get_merged_param_value(j, appDefinition[parametersTag], appParams["parameters"]) - store_value_to_yaml(appDefinition[parametersTag], j, val, paramsetDefinitionComment) - if isEnvSpecificParamset: - storeToEnvSpecificParametersMap(env_specific_params_map, appName, parametersTag, j, val, paramsetName) - writeYamlToFile(applicationParametersFile, appDefinition) - beautifyYaml(applicationParametersFile, application_schema, header_text, wrap_all_strings=False) + appName = appParams["appName"] if "appName" in appParams else appParams["name"] + applicationParametersFile = os.path.dirname(templatePath) + "/Applications/" + appName + ".yml" + appDefinition = getApplicationParametersYaml(appName, applicationParametersFile) + for j in appParams["parameters"]: + # get value with potential merge of dicts + val = get_merged_param_value(j, appDefinition[parametersTag], appParams["parameters"]) + store_value_to_yaml(appDefinition[parametersTag], j, val, paramsetDefinitionComment) + if isEnvSpecificParamset: + storeToEnvSpecificParametersMap(env_specific_params_map, appName, parametersTag, j, val, paramsetName) + writeYamlToFile(applicationParametersFile, appDefinition) + beautifyYaml(applicationParametersFile, application_schema, header_text, wrap_all_strings=False) return -def initParametersStructure(map, key, is_app=False) : + +def initParametersStructure(map, key, is_app=False): if key not in map: map[key] = {} map[key]["deployParameters"] = {} @@ -264,29 +308,35 @@ def initParametersStructure(map, key, is_app=False) : if not is_app: map[key]["applications"] = {} -def storeToEnvSpecificParametersMap(env_specific_params_map, applicationName, parametersTag, paramKey, paramValue, paramsetName) : + +def storeToEnvSpecificParametersMap(env_specific_params_map, applicationName, parametersTag, paramKey, paramValue, + paramsetName): if applicationName: - #todo: nail all 3 parameter tags + # todo: nail all 3 parameter tags if applicationName not in env_specific_params_map["applications"]: initParametersStructure(env_specific_params_map["applications"], applicationName, is_app=True) - env_specific_params_map["applications"][applicationName][parametersTag][paramKey] = { "value" : paramValue, "paramsetName" : paramsetName } + env_specific_params_map["applications"][applicationName][parametersTag][paramKey] = {"value": paramValue, + "paramsetName": paramsetName} else: - env_specific_params_map[parametersTag][paramKey] = { "value" : paramValue, "paramsetName" : paramsetName } + env_specific_params_map[parametersTag][paramKey] = {"value": paramValue, "paramsetName": paramsetName} + def getApplicationParametersYaml(appName, applicationParametersFile): - result = yaml.load("name: \""+appName+"\"\ndeployParameters: {}\ntechnicalConfigurationParameters: {}\n") + result = yaml.load("name: \"" + appName + "\"\ndeployParameters: {}\ntechnicalConfigurationParameters: {}\n") os.makedirs(os.path.dirname(applicationParametersFile), exist_ok=True) if os.path.exists(applicationParametersFile): result = openYaml(applicationParametersFile) return result -def updateEnvSpecificParamsets(env_instances_dir, templateName, templateContent, paramset_map) : + +def updateEnvSpecificParamsets(env_instances_dir, templateName, templateContent, paramset_map): envDefinitionYaml = getEnvDefinition(env_instances_dir) result = {} if "envSpecificParamsets" in envDefinitionYaml["envTemplate"]: if templateName in envDefinitionYaml["envTemplate"]["envSpecificParamsets"]: envSpecificParamsets = envDefinitionYaml["envTemplate"]["envSpecificParamsets"][templateName] - logger.info(f"Attaching env-specific deployment paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") + logger.info( + f"Attaching env-specific deployment paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") templateContent["deployParameterSets"] = templateContent["deployParameterSets"] + envSpecificParamsets for pset in envSpecificParamsets: # Check if paramset exists in paramset_map before accessing it @@ -294,11 +344,13 @@ def updateEnvSpecificParamsets(env_instances_dir, templateName, templateContent, for value in paramset_map[pset]: value["envSpecific"] = True else: - logger.warning(f"Paramset '{pset}' referenced in envSpecificParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") + logger.warning( + f"Paramset '{pset}' referenced in envSpecificParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") if "envSpecificE2EParamsets" in envDefinitionYaml["envTemplate"]: if templateName in envDefinitionYaml["envTemplate"]["envSpecificE2EParamsets"]: envSpecificParamsets = envDefinitionYaml["envTemplate"]["envSpecificE2EParamsets"][templateName] - logger.info(f"Attaching env-specific E2E paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") + logger.info( + f"Attaching env-specific E2E paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") templateContent["e2eParameterSets"] = templateContent["e2eParameterSets"] + envSpecificParamsets for pset in envSpecificParamsets: # Check if paramset exists in paramset_map before accessing it @@ -306,48 +358,70 @@ def updateEnvSpecificParamsets(env_instances_dir, templateName, templateContent, for value in paramset_map[pset]: value["envSpecific"] = True else: - logger.warning(f"Paramset '{pset}' referenced in envSpecificE2EParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") + logger.warning( + f"Paramset '{pset}' referenced in envSpecificE2EParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") if "envSpecificTechnicalParamsets" in envDefinitionYaml["envTemplate"]: if templateName in envDefinitionYaml["envTemplate"]["envSpecificTechnicalParamsets"]: envSpecificParamsets = envDefinitionYaml["envTemplate"]["envSpecificTechnicalParamsets"][templateName] - logger.info(f"Attaching env-specific technical paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") - templateContent["technicalConfigurationParameterSets"] = templateContent["technicalConfigurationParameterSets"] + envSpecificParamsets + logger.info( + f"Attaching env-specific technical paramsets: {dump_as_yaml_format(envSpecificParamsets)} to template {templateName}") + templateContent["technicalConfigurationParameterSets"] = templateContent[ + "technicalConfigurationParameterSets"] + envSpecificParamsets for pset in envSpecificParamsets: # Check if paramset exists in paramset_map before accessing it if pset in paramset_map: for value in paramset_map[pset]: value["envSpecific"] = True else: - logger.warning(f"Paramset '{pset}' referenced in envSpecificTechnicalParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") + logger.warning( + f"Paramset '{pset}' referenced in envSpecificTechnicalParamsets for template '{templateName}' was not found. It may have been skipped due to missing variables.") return result -def processTemplate(templatePath, templateName, env_instances_dir, schema_path, paramset_map, env_specific_params_map, resource_profiles_map=None, header_text="", process_env_specific=True): + +def processTemplate(templatePath, templateName, env_instances_dir, schema_path, paramset_map, env_specific_params_map, + resource_profiles_map=None, header_text="", process_env_specific=True): logger.info(f"Processing template: {templateName} in {templatePath}") templateContent = openYaml(templatePath) if process_env_specific: updateEnvSpecificParamsets(env_instances_dir, templateName, templateContent, paramset_map) - #process deployParameters - templateContent["deployParameters"] = convertParameterSetsToParameters(templatePath, templateContent, "deployParameterSets", "deployParameters", paramset_map, env_specific_params_map, header_text, env_instances_dir) + # process deployParameters + templateContent["deployParameters"] = convertParameterSetsToParameters(templatePath, templateContent, + "deployParameterSets", "deployParameters", + paramset_map, env_specific_params_map, + header_text, env_instances_dir) templateContent["deployParameterSets"] = [] - #process e2eParameters - templateContent["e2eParameters"] = convertParameterSetsToParameters(templatePath, templateContent, "e2eParameterSets", "e2eParameters", paramset_map, env_specific_params_map, header_text, env_instances_dir) + # process e2eParameters + templateContent["e2eParameters"] = convertParameterSetsToParameters(templatePath, templateContent, + "e2eParameterSets", "e2eParameters", + paramset_map, env_specific_params_map, + header_text, env_instances_dir) templateContent["e2eParameterSets"] = [] - #process technicalConfigurationParameters - templateContent["technicalConfigurationParameters"] = convertParameterSetsToParameters(templatePath, templateContent, "technicalConfigurationParameterSets", "technicalConfigurationParameters", paramset_map, env_specific_params_map, header_text, env_instances_dir) + # process technicalConfigurationParameters + templateContent["technicalConfigurationParameters"] = convertParameterSetsToParameters(templatePath, + templateContent, + "technicalConfigurationParameterSets", + "technicalConfigurationParameters", + paramset_map, + env_specific_params_map, + header_text, + env_instances_dir) templateContent["technicalConfigurationParameterSets"] = [] # preparing map for needed resource profiles - if "profile" in templateContent and templateContent["profile"] and "name" in templateContent["profile"] and templateContent["profile"]["name"] : + if "profile" in templateContent and templateContent["profile"] and "name" in templateContent["profile"] and \ + templateContent["profile"]["name"]: rpName = templateContent["profile"]["name"] resource_profiles_map[templateName] = rpName writeYamlToFile(templatePath, templateContent) beautifyYaml(templatePath, schema_path, header_text) return + def process_additional_template_parameters(render_env_dir, source_env_dir, all_instances_dir): # getting additional template variables files from env_definition inventoryYaml = getEnvDefinition(render_env_dir) envDefinitionPath = getEnvDefinitionPath(render_env_dir) - if "sharedTemplateVariables" in inventoryYaml["envTemplate"] and len(inventoryYaml["envTemplate"]["sharedTemplateVariables"]) > 0 : + if "sharedTemplateVariables" in inventoryYaml["envTemplate"] and len( + inventoryYaml["envTemplate"]["sharedTemplateVariables"]) > 0: result = {} for sharedVarFileName in inventoryYaml["envTemplate"]["sharedTemplateVariables"]: sharedVarYamls = findResourcesBottomTop(source_env_dir, all_instances_dir, f"/{sharedVarFileName}") @@ -357,13 +431,17 @@ def process_additional_template_parameters(render_env_dir, source_env_dir, all_i result.update(addedVars) logger.info(f"Shared template variables added from: {yamlPath}: \n{dump_as_yaml_format(addedVars)}") elif len(sharedVarYamls) > 1: - logger.error(f"Duplicate shared template variables with key {sharedVarFileName} found in {all_instances_dir}: \n\t" + ",\n\t".join(str(x) for x in sharedVarYamls)) - raise ReferenceError(f"Duplicate shared template variables with key {sharedVarFileName} found. See logs above.") + logger.error( + f"Duplicate shared template variables with key {sharedVarFileName} found in {all_instances_dir}: \n\t" + ",\n\t".join( + str(x) for x in sharedVarYamls)) + raise ReferenceError( + f"Duplicate shared template variables with key {sharedVarFileName} found. See logs above.") else: - raise ReferenceError(f"Shared template variables with key {sharedVarFileName} not found in {all_instances_dir}") + raise ReferenceError( + f"Shared template variables with key {sharedVarFileName} not found in {all_instances_dir}") # updating additional template variables from map - if "additionalTemplateVariables" not in inventoryYaml["envTemplate"] : + if "additionalTemplateVariables" not in inventoryYaml["envTemplate"]: inventoryYaml["envTemplate"]["additionalTemplateVariables"] = {} else: inventoryVars = inventoryYaml["envTemplate"]["additionalTemplateVariables"] @@ -377,28 +455,25 @@ def process_additional_template_parameters(render_env_dir, source_env_dir, all_i else: logger.info(f"No shared templates variables are defined in: {envDefinitionPath}") -def getTemplateNameFromNamespacePath(namespacePath) : + +def getTemplateNameFromNamespacePath(namespacePath): path = pathlib.Path(namespacePath) return path.parent.name -def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir, env_specific_resource_profile_map, all_instances_dir) : + +def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir, + env_specific_resource_profile_map, all_instances_dir, render_context): paramset_map = createParamsetsMap(parameters_dir) - env_dir = env_template_dir+"/"+env_name + env_dir = env_template_dir + "/" + env_name logger.info(f"Env name: {env_name}") logger.info(f"Env dir: {env_dir}") logger.info(f"Parameters dir: {parameters_dir}") - #const - tenant_schema="schemas/tenant.schema.json" - cloud_schema="schemas/cloud.schema.json" - namespace_schema="schemas/namespace.schema.json" - profiles_schema="schemas/resource-profile.schema.json" - - - #removingAnsibleTrash - yamlFiles = findAllYamlsInDir(env_dir) - for f in yamlFiles : - logger.debug(f"Removing ansible trash from: {f}") - removeAnsibleTrashFromFile(f) + # const + tenant_schema = "schemas/tenant.schema.json" + cloud_schema = "schemas/cloud.schema.json" + namespace_schema = "schemas/namespace.schema.json" + profiles_schema = "schemas/resource-profile.schema.json" + envDefinitionYaml = getEnvDefinition(env_dir) logger.info(getEnvDefinitionPath(env_dir)) templateArtifactName = getTemplateArtifactName(envDefinitionYaml) @@ -407,12 +482,12 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res # pathes tenantTemplatePath = env_dir + "/tenant.yml" cloudTemlatePath = env_dir + "/cloud.yml" - namespaceTemplates = findNamespaces(env_dir) + namespaceTemplates = find_namespaces(env_dir) # env specific parameters map - will be filled with env specific parameters during template processing env_specific_parameters_map = {} env_specific_parameters_map["namespaces"] = {} - #process tenant + # process tenant logger.info(f"Processing tenant: {tenantTemplatePath}") beautifyYaml(tenantTemplatePath, tenant_schema, generated_header_text) @@ -448,7 +523,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res # process namespaces template_namespace_names = [] # iterate through namespace definitions and create namespace parameters - for templatePath in namespaceTemplates : + for templatePath in namespaceTemplates: logger.info(f"Processing namespace: {templatePath}") templateName = getTemplateNameFromNamespacePath(templatePath) template_namespace_names.append(templateName) @@ -467,5 +542,5 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res checkEnvSpecificParametersBySchema(env_dir, env_specific_parameters_map, template_namespace_names) # process resource profiles - processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, env_specific_resource_profile_map, header_text=generated_header_text) - + processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, + env_specific_resource_profile_map, render_context, header_text=generated_header_text) diff --git a/scripts/build_env/generate_config_env.py b/scripts/build_env/generate_config_env.py new file mode 100644 index 00000000..54167899 --- /dev/null +++ b/scripts/build_env/generate_config_env.py @@ -0,0 +1,453 @@ +import os +import re +from contextlib import contextmanager +from datetime import datetime +from pathlib import Path +from typing import Optional + +from deepmerge import always_merger +from envgenehelper import logger, openYaml, readYaml, writeYamlToFile, openFileAsString, copy_path, dumpYamlToStr, \ + create_yaml_processor, find_all_yaml_files_by_stem, ensure_directory, dump_as_yaml_format +from envgenehelper.validation import ensure_valid_fields, ensure_required_keys +from jinja2 import Template, TemplateError +from pydantic import BaseModel, Field +from collections import OrderedDict + +from jinja.jinja import create_jinja_env +from jinja.replace_ansible_stuff import replace_ansible_stuff, escaping_quotation + +yml = create_yaml_processor() + + +class Context(BaseModel): + env: Optional[str] = '' + render_dir: Optional[str] = '' + cloud_passport: OrderedDict = Field(default_factory=OrderedDict) + templates_dir: Optional[Path] = None + output_dir: Optional[str] = '' + cluster_name: Optional[str] = '' + env_definition: OrderedDict = Field(default_factory=OrderedDict) + current_env: OrderedDict = Field(default_factory=OrderedDict) + current_env_dir: Optional[str] = '' + current_env_template: OrderedDict = Field(default_factory=OrderedDict) + tenant: Optional[str] = '' + env_template: OrderedDict = Field(default_factory=OrderedDict) + env_instances_dir: Optional[str] = '' + cloud_passport_file_path: Optional[str] = '' + sd_config: OrderedDict = Field(default_factory=OrderedDict) + sd_file_path: Optional[str] = '' + regdefs: OrderedDict = Field(default_factory=OrderedDict) + appdefs: OrderedDict = Field(default_factory=OrderedDict) + regdef_templates: list = Field(default_factory=list) + appdef_templates: list = Field(default_factory=list) + cloud: Optional[str] = '' + deployer: Optional[str] = '' + render_parameters_dir: Optional[str] = '' + env_vars: OrderedDict = Field(default_factory=OrderedDict) + render_profiles_dir: Optional[str] = '' + + start_time: datetime | None = Field(default=None, exclude=True) + + class Config: + extra = "allow" + validate_assignment = True + + def update(self, data: dict | None = None, **kwargs): + if data: + for key, value in data.items(): + setattr(self, key, value) + for key, value in kwargs.items(): + setattr(self, key, value) + logger.debug(f"Context updated: {data or kwargs}") + + @contextmanager + def use(self): + self.start_time = datetime.now() + logger.debug(f"Enter context at {self.start_time}") + try: + yield self + finally: + logger.debug(f"Final state: {self.dict(exclude_none=True)}") + + def as_dict(self, include_none: bool = False) -> dict: + return self.model_dump(exclude_none=not include_none) + + +class EnvGenerator: + def __init__(self): + self.ctx = Context() + logger.debug(f"EnvGenerator initialized with context: {self.ctx.dict(exclude_none=True)}") + + def set_inventory(self): + inventory_path = Path(self.ctx.env_instances_dir) / "Inventory" / "env_definition.yml" + env_definition = openYaml(filePath=inventory_path, safe_load=True) + logger.info(f"env_definition = {env_definition}") + self.ctx.env_definition = env_definition + + def set_cloud_passport(self): + cloud_passport_file_path = self.ctx.cloud_passport_file_path.strip() + if cloud_passport_file_path: + cloud_passport = openYaml(filePath=cloud_passport_file_path, safe_load=True) + logger.info(f"cloud_passport = {cloud_passport}") + self.ctx.cloud_passport = cloud_passport + + def generate_config(self): + templates_dir = Path(__file__).parent / "templates" + template = create_jinja_env(str(templates_dir)).get_template("env_config.yml.j2") + config = readYaml(text=template.render(self.ctx.as_dict()), safe_load=True) + logger.info(f"config = {config}") + self.ctx.config = config + + def set_env_template(self): + env_template_path_stem = f'{self.ctx.templates_dir}/env_templates/{self.ctx.current_env["env_template"]}' + env_template_path = next(iter(find_all_yaml_files_by_stem(env_template_path_stem)), None) + if not env_template_path: + raise ValueError(f'Template descriptor was not found in {env_template_path_stem}') + + env_template = openYaml(filePath=env_template_path, safe_load=True) + logger.info(f"env_template = {env_template}") + self.ctx.current_env_template = env_template + + def validate_applications(self): + applications = self.ctx.sd_config.get("applications", []) + + for app in applications: + version = app.get("version") + deploy_postfix = app.get("deployPostfix") + + if "version" not in app: + raise ValueError(f"Missing 'version' in application: {app}") + if "deployPostfix" not in app: + raise ValueError(f"Missing 'deployPostfix' in application: {app}") + if not isinstance(version, str): + raise ValueError(f"'version' must be string in application: {app}") + if not isinstance(deploy_postfix, str): + raise ValueError(f"'deployPostfix' must be string in application: {app}") + + logger.info(f"Valid application: {app}") + + def generate_ns_postfix(self, ns, ns_template_path) -> str: + deploy_postfix = ns.get("deploy_postfix") + if deploy_postfix: + ns_template_name = deploy_postfix + else: + # get base name(deploy postfix) without extensions + ns_template_name = self.get_template_name(ns_template_path) + return ns_template_name + + def generate_solution_structure(self): + sd_path_stem = f'{self.ctx.current_env_dir}/Inventory/solution-descriptor/sd' + sd_path = next(iter(find_all_yaml_files_by_stem(sd_path_stem)), None) + solution_structure = {} + if sd_path: + self.ctx.sd_file_path = str(sd_path) + sd_config = openYaml(filePath=sd_path, safe_load=True) + self.ctx.sd_config = sd_config + if "applications" not in sd_config: + raise ValueError("Missing 'applications' key in root") + self.validate_applications() + + namespaces = self.ctx.current_env_template.get("namespaces", []) + postfix_template_map = {} + for ns in namespaces: + namespace_template_path = Template(ns["template_path"]).render(self.ctx.as_dict()) + postfix = self.generate_ns_postfix(ns, namespace_template_path) + postfix_template_map[postfix] = namespace_template_path + + for app in sd_config["applications"]: + app_version = app["version"] + app_name, version = app_version.split(":", 1) + postfix = app["deployPostfix"] + + ns_template_path = postfix_template_map.get(postfix) + ns_name = None + if ns_template_path: + rendered_ns = self.render_from_file_to_obj(ns_template_path) + ns_name = rendered_ns.get("name") + + small_dict = { + app_name: { + postfix: { + "version": version, + "namespace": ns_name + } + } + } + always_merger.merge(solution_structure, small_dict) + + always_merger.merge(self.ctx.current_env, {"solution_structure": solution_structure}) + logger.info(f"Rendered solution_structure: {solution_structure}") + + def render_from_file_to_file(self, src_template_path: str, target_file_path: str): + template = openFileAsString(src_template_path) + template = replace_ansible_stuff(template_str=template, template_path=src_template_path) + rendered = create_jinja_env().from_string(template).render(self.ctx.as_dict()) + logger.info(f"Rendered entity: \n {rendered}") + writeYamlToFile(target_file_path, readYaml(escaping_quotation(rendered))) + + def render_from_file_to_obj(self, src_template_path) -> dict: + template = openFileAsString(src_template_path) + template = replace_ansible_stuff(template_str=template, template_path=src_template_path) + rendered = create_jinja_env().from_string(template).render(self.ctx.as_dict()) + logger.info(f"Rendered entity: \n {rendered}") + return readYaml(escaping_quotation(rendered)) + + def render_from_obj_to_file(self, template, target_file_path): + template = replace_ansible_stuff(template_str=dumpYamlToStr(template)) + rendered = create_jinja_env().from_string(template).render(self.ctx.as_dict()) + logger.info(f"Rendered entity: \n {rendered}") + writeYamlToFile(target_file_path, readYaml(escaping_quotation(rendered))) + + def generate_tenant_file(self): + logger.info(f"Generate Tenant yaml for {self.ctx.tenant}") + tenant_file = f'{self.ctx.current_env_dir}/tenant.yml' + tenant_tmpl_path = self.ctx.current_env_template["tenant"] + self.render_from_file_to_file(Template(tenant_tmpl_path).render(self.ctx.as_dict()), tenant_file) + + def generate_override_template(self, template_override, template_path: Path, name): + if template_override: + logger.info(f"Generate override {template_path.stem} yaml for {name}") + self.render_from_obj_to_file(template_override, template_path) + + def generate_cloud_file(self): + cloud = self.calculate_cloud_name() + cloud_template = self.ctx.current_env_template["cloud"] + current_env_dir = self.ctx.current_env_dir + cloud_file = f'{current_env_dir}/cloud.yml' + context = self.ctx.as_dict() + is_template_override = isinstance(cloud_template, dict) + if is_template_override: + logger.info(f"Generate Cloud yaml for cloud {cloud} using cloud.template_path value") + cloud_tmpl_path = cloud_template["template_path"] + self.render_from_file_to_file(Template(cloud_tmpl_path).render(context), cloud_file) + + template_override = cloud_template.get("template_override") + self.generate_override_template(template_override, Path(f'{current_env_dir}/cloud.yml_override'), cloud) + else: + logger.info(f"Generate Cloud yaml for cloud {cloud}") + self.render_from_file_to_file(Template(cloud_template).render(context), cloud_file) + + def generate_namespace_file(self): + context = self.ctx.as_dict() + namespaces = self.ctx.current_env_template["namespaces"] + for ns in namespaces: + ns_template_path = Template(ns["template_path"]).render(context) + ns_template_name = self.generate_ns_postfix(ns, ns_template_path) + logger.info(f"Generate Namespace yaml for {ns_template_name}") + current_env_dir = self.ctx.current_env_dir + ns_dir = f'{current_env_dir}/Namespaces/{ns_template_name}' + namespace_file = f'{ns_dir}/namespace.yml' + self.render_from_file_to_file(ns_template_path, namespace_file) + + self.generate_override_template(ns.get("template_override"), Path(f'{ns_dir}/namespace.yml_override'), + ns_template_name) + + def calculate_cloud_name(self) -> str: + inv = self.ctx.env_definition["inventory"] + cluster_name = self.ctx.cluster_name + candidates = [ + inv.get("cloudName"), + inv.get("passportCloudName", "").replace("-", "_") if inv.get("passportCloudName") else "", + inv.get("cloudPassport", "").replace("-", "_") if inv.get("cloudPassport") else "", + inv.get("environmentName", "").replace("-", "_"), + f"{cluster_name}_{inv.get('environmentName', '')}".replace("-", "_") + if cluster_name and inv.get("environmentName") else "" + ] + + return next((c for c in candidates if c), "") + + def get_template_name(self, template_path: str) -> str: + template_path = Path(template_path) + return ( + template_path.name + .replace(".yml.j2", "") + .replace(".yaml.j2", "") + ) + + def generate_composite_structure(self): + composite_structure = self.ctx.current_env_template.get("composite_structure") + if composite_structure: + logger.info(f"Generate Composite Structure yaml for {composite_structure}") + current_env_dir = self.ctx.current_env_dir + cs_file = Path(current_env_dir) / "composite_structure.yml" + cs_file.parent.mkdir(parents=True, exist_ok=True) + self.render_from_file_to_file(Template(composite_structure).render(self.ctx.as_dict()), str(cs_file)) + + def get_rendered_target_path(self, template_path: Path) -> Path: + path_str = str(template_path) + path_str = path_str.replace(".yml.j2", ".yml").replace(".yaml.j2", ".yml") + return Path(path_str) + + def generate_paramset_templates(self): + render_dir = Path(self.ctx.render_parameters_dir).resolve() + paramset_templates = self.find_templates(render_dir, ["*.yml.j2", "*.yaml.j2"]) + for template_path in paramset_templates: + template_name = self.get_template_name(template_path) + target_path = self.get_rendered_target_path(template_path) + try: + logger.info(f"Try to render paramset {template_name}") + self.render_from_file_to_file(Template(str(template_path)).render(self.ctx.as_dict()), target_path) + logger.info(f"Successfully generated paramset: {template_name}") + if template_path.exists(): + template_path.unlink() + except TemplateError as e: + logger.warning(f"Skipped paramset {template_name}. Error details: {e}") + if target_path.exists(): + target_path.unlink() + + def find_templates(self, path: str, patterns) -> list[Path]: + path = Path(path) + if not path.exists(): + logger.info(f"Templates directory not found: {path}") + return [] + templates = [] + for pattern in patterns: + for f in path.rglob(pattern): + if f.is_file(): + templates.append(f) + + logger.info(f"Found templates: {templates}") + return templates + + def render_app_defs(self): + for def_tmpl_path in self.ctx.appdef_templates: + app_def_str = openFileAsString(def_tmpl_path) + matches = re.findall( + r'^\s*(name|artifactId|groupId):\s*"([^"]+)"', + app_def_str, + flags=re.MULTILINE, + ) + appdef_meta = dict(matches) + ensure_valid_fields(appdef_meta, ["artifactId", "groupId", "name"]) + group_id = appdef_meta["groupId"] + artifact_id = appdef_meta["artifactId"] + self.ctx.update({ + "app_lookup_key": f"{group_id}:{artifact_id}", + "groupId": group_id, + "artifactId": artifact_id, + }) + app_def_trg_path = f"{self.ctx.current_env_dir}/AppDefs/{appdef_meta.get("name")}.yml" + self.render_from_file_to_file(def_tmpl_path, app_def_trg_path) + + def render_reg_defs(self): + for def_tmpl_path in self.ctx.regdef_templates: + reg_def_str = openFileAsString(def_tmpl_path) + matches = re.findall( + r'^\s*(name):\s*"([^"]+)"', + reg_def_str, + flags=re.MULTILINE, + ) + regdef_meta = dict(matches) + ensure_valid_fields(regdef_meta, ["name"]) + reg_def_trg_path = f"{self.ctx.current_env_dir}/RegDefs/{regdef_meta['name']}.yml" + self.render_from_file_to_file(def_tmpl_path, reg_def_trg_path) + + def set_appreg_def_overrides(self): + output_dir = Path(self.ctx.output_dir) + cluster_name = self.ctx.cluster_name + config_file_name = Path("configuration") / "appregdef_config" + config_file_name_yaml = f"{config_file_name}.yaml" + config_file_name_yml = f"{config_file_name}.yml" + + potential_config_files = [ + output_dir / cluster_name / config_file_name_yaml, + output_dir / cluster_name / config_file_name_yml, + output_dir.parent / config_file_name_yaml, + output_dir.parent / config_file_name_yml, + ] + appregdef_config_paths = [f for f in potential_config_files if f.exists()] + if appregdef_config_paths: + appregdef_config = {} + appregdef_config_path = appregdef_config_paths[0] + try: + appregdef_config = openYaml(appregdef_config_path) + logger.info(f"Overrides applications/registries definitions config found at: {appregdef_config_path}") + except Exception as e: + logger.warning(f"Failed to load config at: {appregdef_config_path}. Error: {e}") + + appdefs = self.ctx.appdefs + regdefs = self.ctx.regdefs + appdefs["overrides"] = appregdef_config.get(appdefs, {}).get("overrides", {}) + regdefs["overrides"] = appregdef_config.get(regdefs, {}).get("overrides", {}) + + def process_app_reg_defs(self): + current_env_dir = self.ctx.current_env_dir + templates_dir = self.ctx.templates_dir + patterns = ["*.yaml.j2", "*.yml.j2", "*.j2", "*.yaml", "*.yml"] + appdef_templates = self.find_templates(f"{templates_dir}/appdefs", patterns) + regdef_templates = self.find_templates(f"{templates_dir}/regdefs", patterns) + self.ctx.appdef_templates = appdef_templates + self.ctx.regdef_templates = regdef_templates + + ensure_directory(Path(current_env_dir).joinpath("AppDefs"), 0o755) + ensure_directory(Path(current_env_dir).joinpath("RegDefs"), 0o755) + self.set_appreg_def_overrides() + self.render_app_defs() + self.render_reg_defs() + + def generate_profiles(self, profile_names: list[str]): + logger.info(f"Start rendering profiles from list: {profile_names}") + render_profiles_dir = self.ctx.render_profiles_dir + profile_templates = self.find_templates(render_profiles_dir, ["*.yaml.j2", "*.yml.j2"]) + for template_path in profile_templates: + template_name = self.get_template_name(template_path) + if template_name in profile_names: + self.render_from_file_to_file(template_path, self.get_rendered_target_path(template_path)) + + def generate_config_env(self, env_name: str, extra_env: dict): + logger.info(f"Starting rendering environment {env_name}. Input params are:\n{dump_as_yaml_format(extra_env)}") + with self.ctx.use(): + all_vars = dict(os.environ) + ci_vars = { + "CI_COMMIT_TAG": all_vars.get("CI_COMMIT_TAG"), + "CI_COMMIT_REF_NAME": all_vars.get("CI_COMMIT_REF_NAME"), + } + self.ctx.update(extra_env) + self.ctx.env_vars.update(ci_vars) + self.set_inventory() + self.set_cloud_passport() + self.generate_config() + + current_env = self.ctx.config["environment"] + self.ctx.current_env = current_env + + self.ctx.cloud = self.calculate_cloud_name() + self.ctx.tenant = current_env.get("tenant", '') + self.ctx.deployer = current_env.get('deployer', '') + logger.info(f"current_env = {current_env}") + + self.ctx.update({ + "ND_CMDB_CONFIG_REF": all_vars.get("CI_COMMIT_SHORT_SHA", "No SHA"), + "ND_CMDB_CONFIG_REF_NAME": all_vars.get("CI_COMMIT_REF_NAME", "No Ref Name"), + "ND_CMDB_CONFIG_TAG": all_vars.get("CI_COMMIT_TAG", "No Ref tag"), + "ND_CDMB_REPOSITORY_URL": all_vars.get("CI_REPOSITORY_URL", "No Ref URL"), + }) + env_template = self.ctx.env_template + if env_template: + self.ctx.update({ + "ND_CMDB_ENV_TEMPLATE": env_template + }) + else: + self.ctx.update({ + "ND_CMDB_ENV_TEMPLATE": self.ctx.current_env["env_template"] + }) + + current_env_dir = f'{self.ctx.render_dir}/{self.ctx.env}' + self.ctx.current_env_dir = current_env_dir + self.set_env_template() + + self.generate_solution_structure() + self.generate_tenant_file() + self.generate_cloud_file() + self.generate_namespace_file() + self.generate_composite_structure() + + env_specific_schema = self.ctx.current_env_template.get("envSpecificSchema") + if env_specific_schema: + copy_path(source_path=env_specific_schema, target_dir=current_env_dir) + self.generate_paramset_templates() + + ensure_required_keys(self.ctx.as_dict(), + required=["templates_dir", "env_instances_dir", "cluster_name", "current_env_dir"]) + self.process_app_reg_defs() + logger.info(f"Rendering of templates for environment {env_name} generation was successful") diff --git a/scripts/build_env/jinja/jinja.py b/scripts/build_env/jinja/jinja.py new file mode 100644 index 00000000..bb291ce8 --- /dev/null +++ b/scripts/build_env/jinja/jinja.py @@ -0,0 +1,50 @@ +from urllib.parse import urlsplit + +from envgenehelper import dumpYamlToStr +from jinja2 import Environment, FileSystemLoader, ChainableUndefined, BaseLoader + + +def create_jinja_env(templates_dir: str = "") -> Environment: + loader = FileSystemLoader(templates_dir) if templates_dir else BaseLoader() + if templates_dir: + env = Environment( + loader=loader, + undefined=ChainableUndefined, + trim_blocks=True, + lstrip_blocks=True, + ) + else: + env = Environment( + undefined=ChainableUndefined, + trim_blocks=True, + lstrip_blocks=True, + ) + JinjaFilters.register(env) + return env + + +def urlsplit_filter(value, part=None): + if not isinstance(value, str): return "" + try: + parts = urlsplit(value) + except ValueError as e: + return f"Invalid url: {e}" + if part: + return getattr(parts, part, "") + return parts + + +JINJA_FILTERS = { + "urlsplit": urlsplit_filter, + "to_nice_yaml": dumpYamlToStr +} + + +class JinjaFilters: + @staticmethod + def register(env: Environment, filters=None): + if filters is None: + filters = JINJA_FILTERS.keys() + for name in filters: + if name in JINJA_FILTERS: + env.filters[name] = JINJA_FILTERS[name] diff --git a/scripts/build_env/jinja/replace_ansible_stuff.py b/scripts/build_env/jinja/replace_ansible_stuff.py new file mode 100644 index 00000000..c9603f04 --- /dev/null +++ b/scripts/build_env/jinja/replace_ansible_stuff.py @@ -0,0 +1,104 @@ +import re +from envgenehelper import logger +from jinja.jinja import JINJA_FILTERS + +general_warn_message = ( + "All Ansible built-in filters (ansible.builtin.*) in this template need to be removed/replaced. " + "Using such filters is no longer allowed. Only standard/custom Jinja2 filters should be used. " + f"List of Jinja2 custom filters: {list(JINJA_FILTERS.keys())}" +) + +incorrect_template_warn_message = ( + "Invalid template: Template was automatically fixed." +) + +underscore_var_warn_message = ( + "Local variables with leading underscores (like {{ _tenant }}) are no longer supported. " + "Use the main variable name instead (e.g. {{ tenant }})." +) + +REPLACEMENTS = [ + # ansible.builtin.to_nice_yaml -> to_nice_yaml + ( + re.compile(r"(\|\s*)ansible\.builtin\.to_nice_yaml(\s*(?:\([^)]*\))?)"), + r"\1to_nice_yaml\2", + "ansible.builtin.to_nice_yaml", + general_warn_message + ), + # lookup('ansible.builtin.env', 'VAR') -> env_vars.VAR + ( + re.compile(r"lookup\(\s*'ansible\.builtin\.env'\s*,\s*'([^']+)'\s*\)"), + r"env_vars.\1", + "ansible.builtin.env lookup", + general_warn_message + ), + # | default('x') -> | default('x', true) + ( + re.compile(r"\|\s*default\((['\"])(.+?)\1\s*\)"), + r"| default(\1\2\1, true)", + "jinja2 default without true", + incorrect_template_warn_message + ), + # {{ _tenant }} -> {{ tenant }} + ( + re.compile(r"{{\s*_(\w+)\s*}}"), + r"{{ \1 }}", + "jinja2 underscore variable", + underscore_var_warn_message + ), +] + + +def replace_ansible_stuff(template_str: str, template_path: str = "") -> str: + template_str = template_str.lstrip() + if template_str.startswith('---'): + template_str = template_str.split('\n', 1)[1] + + for pattern, replacement, name, message in REPLACEMENTS: + for match in pattern.finditer(template_str): + if template_path: + logger.warning( + "[JINJA] Replaced %s in template '%s'. %s", + name, + template_path, + message, + ) + else: + logger.warning( + "[JINJA] Replaced %s in template: %r. %s", + name, + match.string, + message, + ) + logger.warning( + "[JINJA] Pattern: %s | Match: %r -> Replacement: %r", + name, + match.group(0), + pattern.sub(replacement, match.group(0)), + ) + template_str = pattern.sub(replacement, template_str) + + return template_str + + +def escaping_quotation(yaml_text: str) -> str: + def replace_line(line: str) -> str: + if ':' not in line: + return line + + key, value = line.split(':', 1) + val = value.strip() + + if val.startswith('"${') and val.endswith('}"'): + inner = val[1:-1] + if '\\"' in inner: + return line + + escaped_inner = inner.replace('"', '\\"') + return f'{key}: "{escaped_inner}"' + + return line + + lines = yaml_text.splitlines() + fixed_lines = [replace_line(line) for line in lines] + return "\n".join(fixed_lines) diff --git a/scripts/build_env/main.py b/scripts/build_env/main.py index 08caa9f6..bd470711 100644 --- a/scripts/build_env/main.py +++ b/scripts/build_env/main.py @@ -1,12 +1,12 @@ import argparse -import ansible_runner from envgenehelper import * from envgenehelper.deployer import * from build_env import build_env, process_additional_template_parameters from cloud_passport import update_env_definition_with_cloud_name from create_credentials import create_credentials +from generate_config_env import EnvGenerator from resource_profiles import get_env_specific_resource_profiles from pathlib import Path @@ -27,7 +27,7 @@ def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templa delete_dir(render_profiles_dir) render_env_dir = f"{render_dir}/{env_name}" copy_path(f'{source_env_dir}/{INVENTORY_DIR_NAME}', f"{render_env_dir}/{INVENTORY_DIR_NAME}") - # clearing instances dir + # clearing instances dir cleanup_resulting_dir(Path(output_dir) / cluster_name / env_name) # copying parameters from templates and instances check_dir_exist_and_create(f'{render_parameters_dir}/from_template') @@ -46,7 +46,7 @@ def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templa def pre_process_env_before_rendering(render_env_dir, source_env_dir, all_instances_dir): process_additional_template_parameters(render_env_dir, source_env_dir, all_instances_dir) update_env_definition_with_cloud_name(render_env_dir, source_env_dir, all_instances_dir) - copy_path(f"{source_env_dir}/Credentials", f"{render_env_dir}") + copy_path(f"{source_env_dir}/Credentials", f"{render_env_dir}/Credentials") def cleanup_resulting_dir(resulting_dir: Path): @@ -153,35 +153,27 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all logger.debug( f"Created environment context: name='{current_env['name']}', environmentName='{current_env['environmentName']}'") - ansible_vars = {} - ansible_vars["env"] = env_name # Keep as string for file paths - ansible_vars["current_env"] = current_env # Object for Jinja2 templates that need current_env.environmentName - ansible_vars["cluster_name"] = cluster_name - ansible_vars["templates_dir"] = templates_dir - ansible_vars["env_instances_dir"] = getAbsPath(render_env_dir) - ansible_vars["render_dir"] = getAbsPath(render_dir) - ansible_vars["render_parameters_dir"] = getAbsPath(render_parameters_dir) - ansible_vars["template_version"] = g_template_version - ansible_vars["cloud_passport_file_path"] = find_cloud_passport_definition(source_env_dir, all_instances_dir) - ansible_vars["cmdb_url"] = cmdb_url - ansible_vars["output_dir"] = output_dir - logger.info( - f"Starting rendering environment {env_name} with ansible. Input params are:\n{dump_as_yaml_format(ansible_vars)}") - r = ansible_runner.run(playbook=getAbsPath('env-builder/main.yaml'), envvars=ansible_vars, verbosity=2) - if (r.rc != 0): - logger.error(f"Error during ansible execution. Result code is: {r.rc}. Status is: {r.status}") - raise ReferenceError(f"Error during ansible execution. See logs above.") - else: - logger.info(f"Ansible execution status is: {r.status}. Stats is: {r.stats}") - + envvars = {} + envvars["env"] = env_name # Keep as string for file paths + envvars["current_env"] = current_env # Object for Jinja2 templates that need current_env.environmentName + envvars["cluster_name"] = cluster_name + envvars["templates_dir"] = templates_dir + envvars["env_instances_dir"] = getAbsPath(render_env_dir) + envvars["render_dir"] = getAbsPath(render_dir) + envvars["render_parameters_dir"] = getAbsPath(render_parameters_dir) + envvars["template_version"] = g_template_version + envvars["cloud_passport_file_path"] = find_cloud_passport_definition(source_env_dir, all_instances_dir) + envvars["cmdb_url"] = cmdb_url + envvars["output_dir"] = output_dir + envvars["render_profiles_dir"] = render_profiles_dir + render_context = EnvGenerator() + render_context.generate_config_env(env_name, envvars) handle_template_override(render_dir) env_specific_resource_profile_map = get_env_specific_resource_profiles(source_env_dir, all_instances_dir, ENV_SPECIFIC_RESOURCE_PROFILE_SCHEMA) # building env - handle_parameter_container(env_name, cluster_name, templates_dir, all_instances_dir, getAbsPath(render_dir)) - build_env(env_name, source_env_dir, render_parameters_dir, render_dir, render_profiles_dir, - env_specific_resource_profile_map, all_instances_dir) + env_specific_resource_profile_map, all_instances_dir, render_context) resulting_dir = post_process_env_after_rendering(env_name, render_env_dir, source_env_dir, all_instances_dir, output_dir) validate_appregdefs(render_dir, env_name) @@ -271,95 +263,6 @@ def validate_parameter_files(param_files): return errors -def handle_parameter_container(env_name, cluster_name, templates_dir, all_instances_dir, render_dir): - logger.info(f'start handle_parameter_container') - env_dir = get_env_instances_dir(env_name, cluster_name, all_instances_dir) - env_definition_yaml = getEnvDefinition(env_dir) - template_name = env_definition_yaml["envTemplate"]["name"] - logger.info(f'handle "{template_name}" template') - template_file = findYamls(f'{templates_dir}/env_templates', f'{template_name}.y') - - for file in template_file: - template_yml = openYaml(file) - merge_template_parameters(template_yml, templates_dir, True) - namespaces = template_yml["namespaces"] - for namespace in namespaces: - if "parameterContainer" in namespace: - namespace_path_name = namespace["template_path"].split("/")[-1] - namespace_path_name = namespace_path_name.split(".")[0] - namespace_path = f'{render_dir}/{env_name}/Namespaces/{namespace_path_name}/namespace.yml' - namespaces_yml = openYaml(namespace_path) - deployment_parameters = namespaces_yml["deployParameters"] - - for parameterContainer in namespace["parameterContainer"]: - if parameterContainer["source"]["name"] == "": - source_parameters_path = findYamls(f'{templates_dir}/parameters_containers/', - f'source/{parameterContainer["override"]["name"]}.y') - source_file_name = extractNameFromFile(source_parameters_path[0]) - else: - source_file_name = parameterContainer["source"]["name"].split(":")[0] - source_file_version = parameterContainer["source"]["name"].split(":")[1] - source_parameters_path = findYamls(f'{templates_dir}/parameters_containers/', - f'source/{source_file_name}-{source_file_version}.y') - source_parameters_yaml = openYaml(source_parameters_path[0]) - - if "base" in source_parameters_yaml: - for key in source_parameters_yaml["base"]["parameters"]: - merge_dict_key_with_comment(key, deployment_parameters, "value", - source_parameters_yaml["base"]["parameters"][key], - f'# parameterContainer "{source_file_name}", base') - - if "features" in parameterContainer: - for features in parameterContainer["features"]: - for key in source_parameters_yaml["features"][features]["parameters"]: - merge_dict_key_with_comment(key, deployment_parameters, "value", - source_parameters_yaml["features"][features]["parameters"][ - key], - f'# parameterContainer "{source_file_name}", feature "{features}"') - - writeYamlToFile(namespace_path, namespaces_yml) - - -def merge_template_parameters(template_yml, templates_dir, override_source=False): - namespaces = template_yml["namespaces"] - for namespace in namespaces: - if "parameterContainer" in namespace: - for parameterContainer in namespace["parameterContainer"]: - logger.info(f'handle {parameterContainer["source"]["name"]}') - if not ":" in parameterContainer["source"]["name"]: - override_parameters_path = findYamls(f'{templates_dir}/parameters_containers/', - f'override/{parameterContainer["override"]["name"]}.y') - if override_source: - copy_path(override_parameters_path[0], - f'{templates_dir}/parameters_containers/source/{extractNameWithExtensionFromFile(override_parameters_path[0])}') - else: - copy_path(override_parameters_path[0], - f'{templates_dir}/parameters_containers/merge/{extractNameWithExtensionFromFile(override_parameters_path[0])}') - else: - source_file_name = parameterContainer["source"]["name"].split(":")[0] - source_file_version = parameterContainer["source"]["name"].split(":")[1] - source_parameters_path = findYamls(f'{templates_dir}/parameters_containers/', - f'source/{source_file_name}-{source_file_version}.y') - if "override" in parameterContainer: - override_parameters_path = findYamls(f'{templates_dir}/parameters_containers/', - f'override/{parameterContainer["override"]["name"]}.y') - logger.info(f'merge parameters from {override_parameters_path} to {source_parameters_path}') - - yaml_to_override = openYaml(source_parameters_path) - src = openYaml(override_parameters_path) - merge_yaml_into_target(yaml_to_override, '', src) - - source_parameters_path = source_parameters_path[0] - if not override_source: - merged_yaml['name'] = parameterContainer["override"]["artifact_name"] - source_parameters_path = source_parameters_path.replace("/source/", "/merge/") - source_parameters_path = source_parameters_path.replace( - f'/{source_file_name}-{source_file_version}.', - f'/{parameterContainer["override"]["artifact_name"]}.') - - writeYamlToFile(source_parameters_path, yaml_to_override) - - def validate_appregdefs(render_dir, env_name): appdef_dir = f"{render_dir}/{env_name}/AppDefs" regdef_dir = f"{render_dir}/{env_name}/RegDefs" @@ -367,17 +270,17 @@ def validate_appregdefs(render_dir, env_name): if os.path.exists(appdef_dir): appdef_files = findAllYamlsInDir(appdef_dir) if not appdef_files: - print(f"[INFO] No AppDef YAMLs found in {appdef_dir}") + logger.info(f"No AppDef YAMLs found in {appdef_dir}") for file in appdef_files: - print(f"[VALIDATING] AppDef file: {file}") + logger.info(f"AppDef file: {file}") validate_yaml_by_scheme_or_fail(file, "schemas/appdef.schema.json") if os.path.exists(regdef_dir): regdef_files = findAllYamlsInDir(regdef_dir) if not regdef_files: - print(f"[INFO] No RegDef YAMLs found in {regdef_dir}") + logger.info(f"No RegDef YAMLs found in {regdef_dir}") for file in regdef_files: - print(f"[VALIDATING] RegDef file: {file}") + logger.info(f"RegDef file: {file}") validate_yaml_by_scheme_or_fail(file, "schemas/regdef.schema.json") diff --git a/scripts/build_env/parameter_container_handler.py b/scripts/build_env/parameter_container_handler.py deleted file mode 100644 index d8dce668..00000000 --- a/scripts/build_env/parameter_container_handler.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -from os import * - -import ansible_runner -from envgenehelper import * - -from main import merge_template_parameters - - -def build_ci_jobs(): - - if os.getenv('CI_SERVER_NAME') == 'GitLab': - base_dir = os.getenv('CI_PROJECT_DIR') - branch = os.getenv('CI_PROJECT_NAME') - elif os.getenv('GITHUB_ACTIONS') == 'true': - base_dir = os.getenv('GITHUB_WORKSPACE') - branch = os.getenv('GITHUB_REPOSITORY').split('/')[-1] - else: - raise EnvironmentError("Unsupported CI/CD environment") - - logger.info(f'path {base_dir}') - - parameter_set = set() - - all_templates = findAllYamlsInDir(f'{base_dir}/templates/env_templates') - logger.info(f'all_templates {all_templates}') - artifact_definition = openYaml('/build_env/scripts/build_template/template/artifact_definition.yml') - - delete_dir(f'{base_dir}/templates/parameters_containers/source') - delete_dir(f'{base_dir}/templates/parameters_containers/merge') - check_dir_exist_and_create(f'{base_dir}/templates/parameters_containers/source') - check_dir_exist_and_create(f'{base_dir}/templates/parameters_containers/merge') - - for template_path in all_templates: - template_yml = openYaml(template_path) - logger.info(f'template_yml {template_yml}') - if "namespaces" in template_yml: - download_source_files(artifact_definition, base_dir, parameter_set, template_yml) - merge_template_parameters(template_yml, f'{base_dir}/templates') - else: - logger.warning(f'No namespaces found in template: {template_path}') - - all_parameters = findAllYamlsInDir(f'{base_dir}/templates/parameters_containers/merge') - logger.info(f'all_parameters {all_parameters}') - all_parameters = sorted(set(all_parameters)) - - delete_dir(f'{base_dir}/gitlab-ci/prefix_build') - check_dir_exist_and_create(f'{base_dir}/gitlab-ci/prefix_build') - copy_path('/build_env/scripts/build_template/template/parameters_containers_empty.yaml', f'{base_dir}/gitlab-ci/parameters_containers.yaml') - - build_sh = openFileAsString('/build_env/scripts/build_template/template/build.sh') - description = openFileAsString('/build_env/scripts/build_template/template/description.yaml') - parameters_containers = openFileAsString('/build_env/scripts/build_template/template/parameters_containers.yaml') - - logger.info(f"files = {all_parameters}") - - for parameter_path in all_parameters: - logger.info(f'parameter_path {parameter_path}') - parameter_name = extractNameFromFile(parameter_path) - check_dir_exist_and_create(f'{base_dir}/gitlab-ci/prefix_build/{parameter_name}') - copy_path(parameter_path, f'{base_dir}/gitlab-ci/prefix_build/{parameter_name}/{parameter_name}.yml') - new_build_sh = build_sh.replace("@paramater-name", f'{parameter_name}') - writeToFile(f'{base_dir}/gitlab-ci/prefix_build/{parameter_name}/build.sh', new_build_sh) - new_description = description.replace("@paramater-name", f'{parameter_name}') - new_description = new_description.replace("@branch-name", f'{branch}') - writeToFile(f'{base_dir}/gitlab-ci/prefix_build/{parameter_name}/description.yaml', new_description) - - new_parameters_containers = parameters_containers.replace("@paramater-name", f'{parameter_name}') - - jobs = openFileAsString(f'{base_dir}/gitlab-ci/parameters_containers.yaml') - writeToFile(f'{base_dir}/gitlab-ci/parameters_containers.yaml', jobs + new_parameters_containers + '\n\n') - - delete_dir(f'{base_dir}/templates/parameters_containers/merge') - - -def download_source_files(artifact_definition, base_dir, parameter_set, template_yml): - namespaces = template_yml["namespaces"] - for namespace in namespaces: - if "parameterContainer" in namespace: - for parameterContainer in namespace["parameterContainer"]: - source_name = parameterContainer["source"]["name"] - if not source_name in parameter_set: - parameter_set.add(source_name) - logger.info(f'source_name = {source_name}') - ansible_vars = {} - - if ":" in source_name: - ansible_vars["version"] = source_name.split(":")[1] - artifact_definition['artifactId'] = source_name.split(":")[0] - ansible_vars["artifact_definition"] = artifact_definition - - logger.info(f"Starting download parameterContainer {source_name}") - r = ansible_runner.run(playbook='/module/ansible/download_parameters.yaml',envvars=ansible_vars, verbosity=2) - if (r.rc != 0): - logger.error( - f"Error during ansible execution. Result code is: {r.rc}. Status is: {r.status}") - raise ReferenceError(f"Error during ansible execution. See logs above.") - else: - logger.info(f"Ansible execution status is: {r.status}. Stats is: {r.stats}") - - addHeaderToYaml(f'{base_dir}/templates/parameters_containers/source/{artifact_definition["artifactId"]}.yml', f'# The contents of this file is generated from Parameter Container artifact: {source_name}\n# Contents will be overwritten by next generation.\n# Please do not modify this content') - - move_path( - f'{base_dir}/templates/parameters_containers/source/{artifact_definition["artifactId"]}.yml', - f'{base_dir}/templates/parameters_containers/source/{artifact_definition["artifactId"]}-{ansible_vars["version"]}.yml') - - -if __name__ == "__main__": - build_ci_jobs() diff --git a/scripts/build_env/prepare_env_templates.py b/scripts/build_env/prepare_env_templates.py deleted file mode 100644 index 2f1bacf9..00000000 --- a/scripts/build_env/prepare_env_templates.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -from os import * - -import ansible_runner -from envgenehelper import * - -from main import merge_template_parameters - -def prepare_env_templates() - - #Checking the workspace - GitLab or Github - if os.getenv('CI_SERVER_NAME') == 'GitLab': - base_dir = os.getenv('CI_PROJECT_DIR') - branch = os.getenv('CI_PROJECT_NAME') - elif os.getenv('GITHUB_ACTIONS') == 'true': - base_dir = os.getenv('GITHUB_WORKSPACE') - branch = os.getenv('GITHUB_REPOSITORY').split('/')[-1] - else: - raise EnvironmentError("Unsupported CI/CD environment") - - #Checking the info about project dir - logger.info(f'Project path: {base_dir}') - logger.info(f'Project branch: {branch}') - - #Deleting the existing folders - delete_dir(f'{base_dir}/templates/env_templates') - delete_dir(f'{base_dir}/templates/parameters') - delete_dir(f'{base_dir}/templates/resource_profiles') - - #Creating the new ones - check_dir_exist_and_create(f'{base_dir}/templates/env_templates') - check_dir_exist_and_create(f'{base_dir}/templates/parameters') - check_dir_exist_and_create(f'{base_dir}/templates/resource_profiles') - -if __name__ == "__main__": - prepare_env_templates() diff --git a/scripts/build_env/resource_profiles.py b/scripts/build_env/resource_profiles.py index 0b3c88c4..589f84a4 100644 --- a/scripts/build_env/resource_profiles.py +++ b/scripts/build_env/resource_profiles.py @@ -2,6 +2,7 @@ from envgenehelper import * +# TODO unit tests def get_env_specific_resource_profiles(env_dir, instances_dir, rp_schema): result = {} logger.info(f"Finding env specific resource profiles for '{env_dir}' in '{instances_dir}'") @@ -12,7 +13,8 @@ def get_env_specific_resource_profiles(env_dir, instances_dir, rp_schema): logger.info(f"No environment specific resource profiles are defined in {envDefinitionPath}") return result envSepcificResourceProfileNames = inventoryYaml["envTemplate"]["envSpecificResourceProfiles"] - logger.info(f"Environment specific resource profiles for '{envDefinitionPath}' are: \n{dump_as_yaml_format(envSepcificResourceProfileNames)}") + logger.info( + f"Environment specific resource profiles for '{envDefinitionPath}' are: \n{dump_as_yaml_format(envSepcificResourceProfileNames)}") for templateType in envSepcificResourceProfileNames: logger.debug(f"Searching for env specific resource profiles for template '{templateType}'") profileFileName = envSepcificResourceProfileNames[templateType] @@ -24,14 +26,18 @@ def get_env_specific_resource_profiles(env_dir, instances_dir, rp_schema): validate_yaml_by_scheme_or_fail(yamlPath, rp_schema) logger.info(f"Env specific resource profile file for '{profileFileName}' added from: '{yamlPath}'") elif len(resourceProfileFiles) > 1: - logger.error(f"Duplicate resource profile files with key '{profileFileName}' found in '{instances_dir}': \n\t" + ",\n\t".join(str(x) for x in resourceProfileFiles)) - raise ReferenceError(f"Duplicate resource profile files with key '{profileFileName}' found. See logs above.") + logger.error( + f"Duplicate resource profile files with key '{profileFileName}' found in '{instances_dir}': \n\t" + ",\n\t".join( + str(x) for x in resourceProfileFiles)) + raise ReferenceError( + f"Duplicate resource profile files with key '{profileFileName}' found. See logs above.") else: raise ReferenceError(f"Resource profile file with key '{profileFileName}' not found in '{instances_dir}'") logger.info(f"Env specific resource profiles are: \n{dump_as_yaml_format(result)}") return result -def getResourceProfilesFromDir(dir) : + +def getResourceProfilesFromDir(dir): result = {} rpYamls = findAllYamlsInDir(dir) for profileFile in rpYamls: @@ -39,24 +45,28 @@ def getResourceProfilesFromDir(dir) : logger.info(f"Resource profiles in folder {dir}: \n{dump_as_yaml_format(result)}") return result + def get_app_from_resource_profile(appName, profile_yaml): for value in profile_yaml["applications"]: if appName == value["name"]: return value return None + def get_service_from_resource_profile_app(serviceName, app_yaml): for value in app_yaml["services"]: if serviceName == value["name"]: return value return None + def get_param_from_resource_profile_service(paramName, service_yaml): for value in service_yaml["parameters"]: if paramName == value["name"]: return value return None + def merge_resource_profiles(sourceProfileYaml, overrideProfileYaml, overrideProfileName): commentText = f"from {overrideProfileName}" for app in overrideProfileYaml["applications"]: @@ -81,10 +91,12 @@ def merge_resource_profiles(sourceProfileYaml, overrideProfileYaml, overrideProf sourceService["parameters"].append(sourceParam) merge_dict_key_with_comment("name", sourceParam, "name", param, commentText) merge_dict_key_with_comment("value", sourceParam, "value", param, commentText) - else : + else: merge_dict_key_with_comment("value", sourceParam, "value", param, commentText) -def validate_resource_profiles(needed_resource_profiles: dict[str, str], source_profiles: dict[str, str], profiles_schema: str) -> dict[str, str]: + +def validate_resource_profiles(needed_resource_profiles: dict[str, str], source_profiles: dict[str, str], + profiles_schema: str) -> dict[str, str]: profiles_map = {} not_found = '' not_valid = '' @@ -117,8 +129,12 @@ def validate_resource_profiles(needed_resource_profiles: dict[str, str], source_ raise ReferenceError("Not all needed resource profiles found or valid. See logs above.") return profiles_map -def processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, env_specific_resource_profile_map, header_text="") : + +def processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, needed_resource_profiles_map, + env_specific_resource_profile_map, render_context, header_text=""): logger.info(f"Needed profiles map: \n{dump_as_yaml_format(needed_resource_profiles_map)}") + render_context.generate_profiles(set(needed_resource_profiles_map.values())) + render_context.generate_profiles(set(env_specific_resource_profile_map.values())) # map for profiles from templates templateProfilesMap = getResourceProfilesFromDir(resource_profiles_dir) envRpDir = f"{env_dir}/Profiles" @@ -131,15 +147,19 @@ def processResourceProfiles(env_dir, resource_profiles_dir, profiles_schema, nee # iterate through env specific resource profiles and perform override for templateName, envSpecificProfileFile in env_specific_resource_profile_map.items(): if templateName in profilesMap: - logger.info(f"Joining template override profile for namespace '{templateName}' with environment specific profile {envSpecificProfileFile}") + logger.info( + f"Joining template override profile for namespace '{templateName}' with environment specific profile {envSpecificProfileFile}") templateProfileFilePath = profilesMap[templateName] templateProfileYaml = openYaml(templateProfileFilePath) envSpecificProfileYaml = openYaml(envSpecificProfileFile) - merge_resource_profiles(templateProfileYaml, envSpecificProfileYaml, extractNameFromFile(envSpecificProfileFile)) + merge_resource_profiles(templateProfileYaml, envSpecificProfileYaml, + extractNameFromFile(envSpecificProfileFile)) writeYamlToFile(templateProfileFilePath, templateProfileYaml) else: - logger.error(f"No override profile for {templateName} found. Can't apply environment specific resource profile {envSpecificProfileFile}") - raise ReferenceError(f"Can't apply environment specific resource profile for namespace {templateName}. Please set override profile in templates first.") + logger.error( + f"No override profile for {templateName} found. Can't apply environment specific resource profile {envSpecificProfileFile}") + raise ReferenceError( + f"Can't apply environment specific resource profile for namespace {templateName}. Please set override profile in templates first.") # copying source and overriden profiles to resulting dir for profileKey, profileFilePath in profilesMap.items(): logger.debug(f"Copying '{profileKey}' to resulting directory '{envRpDir}'") diff --git a/env-builder/templates/env_config.yml.j2 b/scripts/build_env/templates/env_config.yml.j2 similarity index 54% rename from env-builder/templates/env_config.yml.j2 rename to scripts/build_env/templates/env_config.yml.j2 index 1cca8b86..a5749089 100644 --- a/env-builder/templates/env_config.yml.j2 +++ b/scripts/build_env/templates/env_config.yml.j2 @@ -1,53 +1,53 @@ --- environment: - name: {{env_definition.inventory.environmentName}} - environmentName: {{env_definition.inventory.environmentName}} + name: "{{ env_definition.inventory.environmentName }}" + environmentName: "{{ env_definition.inventory.environmentName }}" {% if env_definition.inventory.tenantName is defined %} - tenant: {{env_definition.inventory.tenantName}} + tenant: "{{ env_definition.inventory.tenantName }}" {% endif %} {% if env_definition.inventory.cloudName is defined %} - cloud: {{env_definition.inventory.cloudName}} + cloud: "{{ env_definition.inventory.cloudName }}" {% elif env_definition.inventory.passportCloudName is defined %} - cloud: {{ env_definition.inventory.passportCloudName | replace("-","_") }} + cloud: "{{ env_definition.inventory.passportCloudName | replace('-','_') }}" {% elif env_definition.inventory.cloudPassport is defined %} - cloud: {{ env_definition.inventory.cloudPassport | replace("-","_") }} + cloud: "{{ env_definition.inventory.cloudPassport | replace('-','_') }}" {% else %} - cloud: {{env_definition.inventory.environmentName | replace("-","_") }} + cloud: "{{ env_definition.inventory.environmentName | replace('-','_') }}" {% endif %} {% if env_definition.inventory.cloudName is defined %} - cloudNameWithCluster: {{env_definition.inventory.cloudName}} + cloudNameWithCluster: "{{ env_definition.inventory.cloudName }}" {% elif env_definition.inventory.passportCloudName is defined %} - cloudNameWithCluster: {{ (env_definition.inventory.passportCloudName ~ '_' ~ env_definition.inventory.environmentName) | replace("-","_") }} + cloudNameWithCluster: "{{ (env_definition.inventory.passportCloudName ~ '_' ~ env_definition.inventory.environmentName) | replace('-','_') }}" {% elif env_definition.inventory.cloudPassport is defined %} - cloudNameWithCluster: {{ (env_definition.inventory.cloudPassport ~ '_' ~ env_definition.inventory.environmentName) | replace("-","_") }} + cloudNameWithCluster: "{{ (env_definition.inventory.cloudPassport ~ '_' ~ env_definition.inventory.environmentName) | replace('-','_') }}" {% else %} - cloudNameWithCluster: {{ (cluster_name ~ '_' ~ env_definition.inventory.environmentName) | replace("-","_") }} + cloudNameWithCluster: "{{ (cluster_name ~ '_' ~ env_definition.inventory.environmentName) | replace('-','_') }}" {% endif %} {% if env_definition.inventory.deployer is defined %} - deployer: {{env_definition.inventory.deployer}} - cmdb_url: {{cmdb_url}} - cmdb_name: {{env_definition.inventory.deployer}} + deployer: "{{ env_definition.inventory.deployer }}" + cmdb_url: "{{ cmdb_url }}" + cmdb_name: "{{ env_definition.inventory.deployer }}" {% endif %} {% if env_definition.inventory.description is defined %} - description: {{env_definition.inventory.description}} + description: "{{ env_definition.inventory.description }}" {% else %} description: "" {% endif %} {% if env_definition.inventory.owners is defined %} - owners: {{env_definition.inventory.owners}} + owners: "{{ env_definition.inventory.owners }}" {% else %} owners: "" {% endif %} - env_template: {{env_definition.envTemplate.name}} + env_template: "{{ env_definition.envTemplate.name }}" {% if env_definition.envTemplate.envSpecificParamsets is defined %} envSpecificParamsets: -{{env_definition.envTemplate.envSpecificParamsets | ansible.builtin.to_nice_yaml | indent(4, true) }} +{{ env_definition.envTemplate.envSpecificParamsets | to_nice_yaml | indent(4, true) }} {% else %} envSpecificParamsets: {} {% endif %} {% if env_definition.envTemplate.additionalTemplateVariables is defined %} additionalTemplateVariables: -{{env_definition.envTemplate.additionalTemplateVariables | ansible.builtin.to_nice_yaml | indent(4, true) }} +{{ env_definition.envTemplate.additionalTemplateVariables | to_nice_yaml | indent(4, true) }} {% else %} additionalTemplateVariables: {} {% endif %} @@ -65,7 +65,7 @@ environment: {% endif %} {% if cloud_passport is defined %} cloud_passport: -{{cloud_passport | ansible.builtin.to_nice_yaml | indent(4, true) }} +{{ cloud_passport | to_nice_yaml | indent(4, true) }} {% else %} cloud_passport: {} {% endif %} diff --git a/scripts/build_env/test_handle_sd.py b/scripts/build_env/test_handle_sd.py index 370402fb..408ad268 100644 --- a/scripts/build_env/test_handle_sd.py +++ b/scripts/build_env/test_handle_sd.py @@ -1,5 +1,4 @@ import os -from pathlib import Path import pytest from ruamel.yaml import YAML diff --git a/scripts/build_env/test_helper.py b/scripts/build_env/test_helper.py index b5f565ff..cec061f3 100644 --- a/scripts/build_env/test_helper.py +++ b/scripts/build_env/test_helper.py @@ -33,7 +33,7 @@ def assert_dirs_content(source_dir, target_dir, missed_files=False, extra_files= missing_files = source_files - target_files assert not missing_files, f"Missing files in target: {dump_as_yaml_format([source_map[name] for name in missing_files])}" - files_to_compare = [os.path.basename(f) for f in get_all_files_in_dir(source_dir)] + files_to_compare = get_all_files_in_dir(source_dir) match, mismatch, errors = filecmp.cmpfiles(source_dir, target_dir, files_to_compare, shallow=False) logger.info(f"Match: {dump_as_yaml_format(match)}") diff --git a/scripts/build_env/test_render_envs.py b/scripts/build_env/test_render_envs.py index 312b1e44..92d9b0de 100644 --- a/scripts/build_env/test_render_envs.py +++ b/scripts/build_env/test_render_envs.py @@ -1,12 +1,10 @@ -import difflib -import filecmp from os import environ -from pathlib import Path import pytest from envgenehelper import * from main import render_environment, cleanup_resulting_dir +from test_helper import TestHelpers test_data = [ # (cluster_name, environment_name, template) @@ -15,15 +13,16 @@ ("cluster-01", "env-03", "composite-dev"), ("cluster-01", "env-04", "simple"), ("cluster01", "env01", "test-01"), - ("cluster01", "env01", "test-01"), ("cluster01", "env03", "test-template-1"), ("cluster01", "env04", "test-template-2") ] -g_templates_dir = getAbsPath("../../test_data/test_templates") -g_inventory_dir = getAbsPath("../../test_data/test_environments") -g_output_dir = getAbsPath("../../tmp/test_environments") +base_dir = Path(__file__).parent.resolve() +g_templates_dir = str((base_dir / "../../test_data/test_templates").resolve()) +g_inventory_dir = str((base_dir / "../../test_data/test_environments").resolve()) +g_output_dir = str((base_dir / "../../tmp/test_environments").resolve()) g_base_dir = get_parent_dir_for_dir(g_inventory_dir) +environ['CI_PROJECT_DIR'] = g_base_dir @pytest.fixture(autouse=True) @@ -31,43 +30,14 @@ def change_test_dir(request, monkeypatch): monkeypatch.chdir(request.fspath.dirname + "/../..") -# TODO use func from test_helper during impl replacement ansible @pytest.mark.parametrize("cluster_name, env_name, version", test_data) def test_render_envs(cluster_name, env_name, version): - environ['CI_PROJECT_DIR'] = g_base_dir render_environment(env_name, cluster_name, g_templates_dir, g_inventory_dir, g_output_dir, version, g_base_dir) source_dir = f"{g_inventory_dir}/{cluster_name}/{env_name}" generated_dir = f"{g_output_dir}/{cluster_name}/{env_name}" - files_to_compare = get_all_files_in_dir(source_dir, source_dir + "/") + files_to_compare = get_all_files_in_dir(source_dir) logger.info(dump_as_yaml_format(files_to_compare)) - match, mismatch, errors = filecmp.cmpfiles(source_dir, generated_dir, files_to_compare, shallow=False) - logger.info(f"Match: {dump_as_yaml_format(match)}") - if len(mismatch) > 0: - logger.error(f"Mismatch: {dump_as_yaml_format(mismatch)}") - for file in mismatch: - file1 = os.path.join(source_dir, file) - file2 = os.path.join(generated_dir, file) - try: - with open(file1, 'r') as f1, open(file2, 'r') as f2: - diff = difflib.unified_diff( - f1.readlines(), - f2.readlines(), - fromfile=file1, - tofile=file2, - lineterm='' - ) - diff_text = '\n'.join(diff) - logger.error(f"Diff for {file}:\n{diff_text}") - except Exception as e: - logger.error(f"Could not read files for diff: {file1}, {file2}. Error: {e}") - else: - logger.info(f"Mismatch: {dump_as_yaml_format(mismatch)}") - if len(errors) > 0: - logger.fatal(f"Errors: {dump_as_yaml_format(errors)}") - else: - logger.info(f"Errors: {dump_as_yaml_format(errors)}") - assert len(mismatch) == 0, f"Files from source and rendering result mismatch: {dump_as_yaml_format(mismatch)}" - assert len(errors) == 0, f"Error during comparing source and rendering result: {dump_as_yaml_format(errors)}" + TestHelpers.assert_dirs_content(source_dir, generated_dir, True, False) def setup_test_dir(tmp_path): diff --git a/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml b/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml index 364ae889..abaf90cd 100644 --- a/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml +++ b/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml @@ -2,22 +2,22 @@ cloud-deploy-sa-token: type: "secret" data: secret: "eyJhbGciOiJSUzI1NiIsImtpZCI6Ilh6bE9fS19OaC1LZ2ZJNk5fVXlIV18yWFh1V19oYlJqV2FfR2JtY3MifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6ImRlZmF1bHQtdG9rZW4teHh4eHgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGVmYXVsdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjEyMzQ1Ni0xMjM0LTEyMzQtMTIzNC0xMjM0NTY3ODkiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6ZGVmYXVsdDpkZWZhdWx0In0.AB12CD34EF56GH78IJ90KL12MN34OP56QR78ST90UV12WX34YZ56AB78CD90EF12GH34IJ56KL78MN90OP12QR34ST56UV78WX90YZ12AB34CD56EF78GH90IJ12KL34MN56OP78QR90ST12UV34WX56YZ78AB90CD12EF34GH56IJ78KL90MN12OP34QR56ST78UV90WX12YZ34AB56CD78EF90GH12IJ34KL56MN78OP90QR12ST34UV56WX78YZ90" +consul-cred: + type: "secret" + data: + secret: "b2c4d6e8-f1a3-5b7d-9e0f-1a2b3c4d5e6f" dbaas-cred: - type: usernamePassword + type: "usernamePassword" data: - username: dbaas-username - password: dbaas-password + username: "dbaas-username" + password: "dbaas-password" maas-cred: - type: usernamePassword - data: - username: maas-username - password: maas-password -consul-cred: - type: secret + type: "usernamePassword" data: - secret: b2c4d6e8-f1a3-5b7d-9e0f-1a2b3c4d5e6f + username: "maas-username" + password: "maas-password" minio-cred: - type: usernamePassword + type: "usernamePassword" data: - username: minio-username - password: minio-password + username: "minio-username" + password: "minio-password" diff --git a/test_data/test_environments/cluster-01/env-01/AppDefs/application-1.yml b/test_data/test_environments/cluster-01/env-01/AppDefs/application-1.yml new file mode 100644 index 00000000..cf96b356 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-01/AppDefs/application-1.yml @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "registry-1" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml b/test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml new file mode 100644 index 00000000..3a1b2389 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml @@ -0,0 +1,7 @@ +name: "application-2" +registryName: "registry-2" +artifactId: "application-2" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/cluster-01/env-01/Credentials/credentials.yml b/test_data/test_environments/cluster-01/env-01/Credentials/credentials.yml index a3c1994b..bcf8c55b 100644 --- a/test_data/test_environments/cluster-01/env-01/Credentials/credentials.yml +++ b/test_data/test_environments/cluster-01/env-01/Credentials/credentials.yml @@ -109,6 +109,11 @@ integration-cred: # namespace env-01-bss data: username: "envgeneNullValue" # FillMe password: "envgeneNullValue" # FillMe +keycloak-admin-cred: # namespace env-01-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe kms-cert: # namespace env-01-bss type: "secret" data: @@ -123,6 +128,21 @@ minio-cred: # cloud passport: cluster-01 version: 1.5 data: username: "minio-username" password: "minio-password" +oracle-nrm-cred: # namespace env-01-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +postgresql-cred: # namespace env-01-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +s3-dbp-bucket-cred: # namespace env-01-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe service-integration-cred: # shared credentials: integration-creds type: "usernamePassword" data: diff --git a/test_data/test_environments/cluster-01/env-01/Inventory/parameters/nrm.yml b/test_data/test_environments/cluster-01/env-01/Inventory/parameters/nrm.yml new file mode 100644 index 00000000..00c1923b --- /dev/null +++ b/test_data/test_environments/cluster-01/env-01/Inventory/parameters/nrm.yml @@ -0,0 +1,44 @@ +version: 23.4 +name: nrm +parameters: + nrm: + apps: + rb: + volumes: + outputs: + capacity: 20Gi + dbp: + storage: + bucketName: dbp-test-bucket + oracle: + rw: + url: oracle://${creds.get("oracle-nrm-cred").username}:${creds.get("oracle-nrm-cred").password}@testlike1-nrm-db-1.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:1521/NRS1D1 + kafka: + username: "" + address: "testlike1-ocs-kafka-zk-1.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092;testlike1-ocs-kafka-zk-2.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092;testlike1-ocs-kafka-zk-3.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092" + password: "" + identityProviders: + infra-keycloak: + caBundle: | + -----BEGIN CERTIFICATE----- + MIIDoDCCAoigAwIBAgIUJ307ifcRysSpk2XoYdjaFdwPnT8wDQYJKoZIhvcNAQEL + BQAwaDELMAkGA1UEBhMCS1oxDzANBgNVBAgTBkFsbWF0eTEPMA0GA1UEBxMGQWxt + YXR5MRMwEQYDVQQKEwpOZXRjcmFja2VyMQ0wCwYDVQQLEwRTRFNJMRMwEQYDVQQD + EwpQcml2YXRlIENBMB4XDTI0MDQwMzEzNDUwMFoXDTI5MDQwMjEzNDUwMFowaDEL + MAkGA1UEBhMCS1oxDzANBgNVBAgTBkFsbWF0eTEPMA0GA1UEBxMGQWxtYXR5MRMw + EQYDVQQKEwpOZXRjcmFja2VyMQ0wCwYDVQQLEwRTRFNJMRMwEQYDVQQDEwpQcml2 + YXRlIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1dv9ptwuhf3z + LqHjjOn/cSvUEkkCj1zPxL/YZcxEB4nrNTnOSJxZaNcKxdOS4dThzllxmVk9r52F + dRbLtW2A3MR9u6lFiisn1shCbvmslofm/2mmrY+OkA9O1+yIJ9V6i2aIodoLawCw + n24bhZkA8iIngEK/0+85GHMF9bDnMUFFaEat77EsFWDkl1AaGeJvcZ3/29dAK2nT + RC9f6kcgp6BTsTu0safPzr4HJxV8v172SaVsAnOyQmnWGnzwlBHfju+Vokq9U/U1 + BckvOrJceXhBlbcZLYl5YgavWxkVsF/HzPbqExDNTb7e6T8DWMYMD3ZOXyd89qi+ + lZirJS5aDwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB + /zAdBgNVHQ4EFgQUEg7VbkMQFZ5dofdDd+pAZhkryNswDQYJKoZIhvcNAQELBQAD + ggEBAJ+UCNTTlFBHVctJuiqIZWes4jdAWuSiljtf8kgB06D2uVJbZKYlrCUViasV + Hd5YIaD4olFNEbJmNVgedWg3O9AG3dcluXphdA7tbrbOW8tm2GBp42J0Z9Nh5ZDN + +8+hJmS0D69uc1o4ExIteyQZby/6qdtfHrdXVYR46pXmvcDUAcsA/4xtf2sJZyxe + /FzeCydVAVCGiYj1JUSvKTKss3TKE5Ku3auLswg1A2ZaSWT/+3ob3mBCZBfmAszB + Qs1rf1augRmFA1TKh6iQtEMV0dk0y5ZnperTJnX6kNzTb8VXhPNaCGl5iMLKhIn6 + sEuDrte505pPOOyZCggQMQBE+tk= + -----END CERTIFICATE----- diff --git a/test_data/test_environments/cluster-01/env-01/Namespaces/billing/namespace.yml b/test_data/test_environments/cluster-01/env-01/Namespaces/billing/namespace.yml index 007078ee..5db44629 100644 --- a/test_data/test_environments/cluster-01/env-01/Namespaces/billing/namespace.yml +++ b/test_data/test_environments/cluster-01/env-01/Namespaces/billing/namespace.yml @@ -13,8 +13,8 @@ profile: name: "prod_billing_override" baseline: "prod" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" PARAM_2: "value-2" # paramset: billing version: 25.1 source: template PARAM_6: "value-6" # paramset: billing version: 25.1 source: template e2eParameters: {} @@ -22,5 +22,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-01/Namespaces/bss/namespace.yml b/test_data/test_environments/cluster-01/env-01/Namespaces/bss/namespace.yml index e37274e0..9afa7b34 100644 --- a/test_data/test_environments/cluster-01/env-01/Namespaces/bss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-01/Namespaces/bss/namespace.yml @@ -13,13 +13,16 @@ profile: name: "prod_bss_override" baseline: "prod" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + DEPLOY_W_HELM: true # paramset: nrm version: 23.4 source: template + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" + ESCAPE_SEQUENCE: true # paramset: nrm version: 23.4 source: template KMS_CERT_IN_BASE64: "${envgen.creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template OVERRIDE_PARAMETER: "template-value" PARAM_2: "value-2" # paramset: bss version: 25.1 source: template PARAM_6: "value-6" # paramset: bss version: 25.1 source: template SITE_LEVEL_PARAM_1: "site-level-value-1" # paramset: prod-shared version: 25.1 source: instance + STORAGE_USERNAME: "${creds.get(\"minio-cred\").username}" TEST_CREDS_GET_PASSWORD_PASSWORD: "${envgen.creds.get('creds-get-password-cred').password}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_PASSWORD_USERNAME: "${envgen.creds.get('creds-get-username-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_SECRET_PARAM: "${envgen.creds.get('creds-get-secret').secret}" # paramset: test-deploy-creds version: 1 source: template @@ -37,7 +40,72 @@ deployParameters: TEST_SHARED_CREDS: "${envgen.creds.get('integration-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_SHARED_CREDS_ACTIVATOR: "${envgen.creds.get('service-integration-cred').password}" # paramset: test-deploy-creds version: 1 source: template bss-app-exist: true - 'oss_ns': "env-01-oss" + nrm: # paramset: nrm version: 23.4 source: instance + apps: + dbp: + storage: + basePath: "dbp-docs" + bucketName: "dbp-test-bucket" + enabled: true + rb: + volumes: + outputs: + capacity: "20Gi" + storageClass: "csi-cinder-sc-retain" + identityProviders: + infra-keycloak: + caBundle: | + -----BEGIN CERTIFICATE----- + MIIDoDCCAoigAwIBAgIUJ307ifcRysSpk2XoYdjaFdwPnT8wDQYJKoZIhvcNAQEL + BQAwaDELMAkGA1UEBhMCS1oxDzANBgNVBAgTBkFsbWF0eTEPMA0GA1UEBxMGQWxt + YXR5MRMwEQYDVQQKEwpOZXRjcmFja2VyMQ0wCwYDVQQLEwRTRFNJMRMwEQYDVQQD + EwpQcml2YXRlIENBMB4XDTI0MDQwMzEzNDUwMFoXDTI5MDQwMjEzNDUwMFowaDEL + MAkGA1UEBhMCS1oxDzANBgNVBAgTBkFsbWF0eTEPMA0GA1UEBxMGQWxtYXR5MRMw + EQYDVQQKEwpOZXRjcmFja2VyMQ0wCwYDVQQLEwRTRFNJMRMwEQYDVQQDEwpQcml2 + YXRlIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1dv9ptwuhf3z + LqHjjOn/cSvUEkkCj1zPxL/YZcxEB4nrNTnOSJxZaNcKxdOS4dThzllxmVk9r52F + dRbLtW2A3MR9u6lFiisn1shCbvmslofm/2mmrY+OkA9O1+yIJ9V6i2aIodoLawCw + n24bhZkA8iIngEK/0+85GHMF9bDnMUFFaEat77EsFWDkl1AaGeJvcZ3/29dAK2nT + RC9f6kcgp6BTsTu0safPzr4HJxV8v172SaVsAnOyQmnWGnzwlBHfju+Vokq9U/U1 + BckvOrJceXhBlbcZLYl5YgavWxkVsF/HzPbqExDNTb7e6T8DWMYMD3ZOXyd89qi+ + lZirJS5aDwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB + /zAdBgNVHQ4EFgQUEg7VbkMQFZ5dofdDd+pAZhkryNswDQYJKoZIhvcNAQELBQAD + ggEBAJ+UCNTTlFBHVctJuiqIZWes4jdAWuSiljtf8kgB06D2uVJbZKYlrCUViasV + Hd5YIaD4olFNEbJmNVgedWg3O9AG3dcluXphdA7tbrbOW8tm2GBp42J0Z9Nh5ZDN + +8+hJmS0D69uc1o4ExIteyQZby/6qdtfHrdXVYR46pXmvcDUAcsA/4xtf2sJZyxe + /FzeCydVAVCGiYj1JUSvKTKss3TKE5Ku3auLswg1A2ZaSWT/+3ob3mBCZBfmAszB + Qs1rf1augRmFA1TKh6iQtEMV0dk0y5ZnperTJnX6kNzTb8VXhPNaCGl5iMLKhIn6 + sEuDrte505pPOOyZCggQMQBE+tk= + -----END CERTIFICATE----- + credentials: + password: "${creds.get(\"keycloak-admin-cred\").password}" + username: "${creds.get(\"keycloak-admin-cred\").username}" + url: "https://infra-keycloak-infra-keycloak./realms/nrm-env-01" + usedFor: + - "implicit" + kafka: + address: "testlike1-ocs-kafka-zk-1.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092;testlike1-ocs-kafka-zk-2.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092;testlike1-ocs-kafka-zk-3.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:9092" + password: "" + username: "" + monitoring: + grafana: + url: "https://grafana-platform-monitoring./" + namespace: "platform-monitoring" + oracle: + rw: + url: "oracle://${creds.get(\"oracle-nrm-cred\").username}:${creds.get(\"oracle-nrm-cred\").password}@testlike1-nrm-db-1.cyta-cloud25mdc-cloud1.cyta.openshift.sdntest.netcracker.com:1521/NRS1D1" + postgresql: + password: "${creds.get(\"postgresql-cred\").password}" + ro: + address: "pg-patroni.postgresql:5432" + rw: + address: "pg-patroni.postgresql:5432" + username: "${creds.get(\"postgresql-cred\").username}" + prometheus-adapter: + install: "true" + storage: + url: "s3://${creds.get(\"s3-dbp-bucket-cred\").username}:${creds.get(\"s3-dbp-bucket-cred\").password}@s3.eu-central-1.amazonaws.com" + oss_ns: "env-01-oss" e2eParameters: E2E_PARAM_1: "e2e-param-val-1" E2E_PARAM_2: "e2e-param-val-2" diff --git a/test_data/test_environments/cluster-01/env-01/Namespaces/core/namespace.yml b/test_data/test_environments/cluster-01/env-01/Namespaces/core/namespace.yml index 07987f9e..7e54b9f3 100644 --- a/test_data/test_environments/cluster-01/env-01/Namespaces/core/namespace.yml +++ b/test_data/test_environments/cluster-01/env-01/Namespaces/core/namespace.yml @@ -20,4 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_environments/cluster-01/env-01/Namespaces/oss/namespace.yml b/test_data/test_environments/cluster-01/env-01/Namespaces/oss/namespace.yml index 7c1f224f..639eb325 100644 --- a/test_data/test_environments/cluster-01/env-01/Namespaces/oss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-01/Namespaces/oss/namespace.yml @@ -20,5 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml b/test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml new file mode 100644 index 00000000..599de06d --- /dev/null +++ b/test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml @@ -0,0 +1,19 @@ +name: "registry-1" +credentialsId: "registry-cred" +mavenConfig: + repositoryDomainName: "maven.qubership.org" + fullRepositoryUrl: "https://maven.qubership.org/repository" + targetSnapshot: "snapshot" + targetStaging: "staging" + targetRelease: "release" + releaseGroup: "" + snapshotGroup: "" +dockerConfig: + snapshotUri: "docker.qubership.org/snapshot" + stagingUri: "docker.qubership.org/staging" + releaseUri: "docker.qubership.org/release" + groupUri: "docker.qubership.org/group" + snapshotRepoName: "docker-snapshot" + stagingRepoName: "docker-staging" + releaseRepoName: "docker-release" + groupName: "docker-group" diff --git a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml b/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml new file mode 100644 index 00000000..ef136e37 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml @@ -0,0 +1,19 @@ +name: "registry-2" +credentialsId: "registry-cred" +mavenConfig: + repositoryDomainName: "maven.qubership.org" + fullRepositoryUrl: "https://maven.qubership.org/repository" + targetSnapshot: "snapshot" + targetStaging: "staging" + targetRelease: "release" + releaseGroup: "" + snapshotGroup: "" +dockerConfig: + snapshotUri: "docker.qubership.org/snapshot" + stagingUri: "docker.qubership.org/staging" + releaseUri: "docker.qubership.org/release" + groupUri: "docker.qubership.org/group" + snapshotRepoName: "docker-snapshot" + stagingRepoName: "docker-staging" + releaseRepoName: "docker-release" + groupName: "docker-group" diff --git a/test_data/test_environments/cluster-01/env-01/tenant.yml b/test_data/test_environments/cluster-01/env-01/tenant.yml index d890d954..c5dd3f11 100644 --- a/test_data/test_environments/cluster-01/env-01/tenant.yml +++ b/test_data/test_environments/cluster-01/env-01/tenant.yml @@ -16,4 +16,3 @@ globalE2EParameters: environmentParameters: {} deployParameters: ESCAPE_SEQUENCE: "true" - diff --git a/test_data/test_environments/cluster-01/env-02/Credentials/credentials.yml b/test_data/test_environments/cluster-01/env-02/Credentials/credentials.yml index 0bc8106a..1e5e9e35 100644 --- a/test_data/test_environments/cluster-01/env-02/Credentials/credentials.yml +++ b/test_data/test_environments/cluster-01/env-02/Credentials/credentials.yml @@ -104,6 +104,11 @@ integration-cred: # namespace env-02-bss data: username: "envgeneNullValue" # FillMe password: "envgeneNullValue" # FillMe +keycloak-admin-cred: # namespace env-02-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe kms-cert: # namespace env-02-bss type: "secret" data: @@ -118,6 +123,21 @@ minio-cred: # cloud passport: cluster-01 version: 1.5 data: username: "minio-username" password: "minio-password" +ocs-kafka-cred: # namespace env-02-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +postgresql-cred: # namespace env-02-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +s3-dbp-bucket-cred: # namespace env-02-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe service-integration-cred: # namespace env-02-bss type: "usernamePassword" data: diff --git a/test_data/test_environments/cluster-01/env-02/Namespaces/billing/namespace.yml b/test_data/test_environments/cluster-01/env-02/Namespaces/billing/namespace.yml index 8e89067d..7c6ffd15 100644 --- a/test_data/test_environments/cluster-01/env-02/Namespaces/billing/namespace.yml +++ b/test_data/test_environments/cluster-01/env-02/Namespaces/billing/namespace.yml @@ -10,11 +10,11 @@ labels: cleanInstallApprovalRequired: false mergeDeployParametersAndE2EParameters: false profile: - name: "dev_billing_override" + name: "rgrs_oc_profile" baseline: "dev" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" PARAM_2: "value-2" # paramset: billing version: 25.1 source: template PARAM_6: "value-6" # paramset: billing version: 25.1 source: template e2eParameters: {} @@ -22,5 +22,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-02/Namespaces/bss/namespace.yml b/test_data/test_environments/cluster-01/env-02/Namespaces/bss/namespace.yml index 08ca89a1..1a9577af 100644 --- a/test_data/test_environments/cluster-01/env-02/Namespaces/bss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-02/Namespaces/bss/namespace.yml @@ -13,13 +13,16 @@ profile: name: "dev_bss_override" baseline: "dev" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + DEPLOY_W_HELM: true # paramset: nrm version: 23.4 source: template + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" + ESCAPE_SEQUENCE: true # paramset: nrm version: 23.4 source: template KMS_CERT_IN_BASE64: "${envgen.creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template OVERRIDE_PARAMETER: "env-specific-value" # paramset: env-specific-bss version: 23.1 source: instance PARAM_2: "value-2" # paramset: bss version: 25.1 source: template PARAM_6: "value-6" # paramset: bss version: 25.1 source: template SITE_LEVEL_PARAM_1: "site-level-value-1" # paramset: prod-shared version: 25.1 source: instance + STORAGE_USERNAME: "${creds.get(\"minio-cred\").username}" TEST_CREDS_GET_PASSWORD_PASSWORD: "${envgen.creds.get('creds-get-password-cred').password}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_PASSWORD_USERNAME: "${envgen.creds.get('creds-get-username-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_SECRET_PARAM: "${envgen.creds.get('creds-get-secret').secret}" # paramset: test-deploy-creds version: 1 source: template @@ -37,6 +40,43 @@ deployParameters: TEST_SHARED_CREDS: "${envgen.creds.get('integration-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_SHARED_CREDS_ACTIVATOR: "${envgen.creds.get('service-integration-cred').password}" # paramset: test-deploy-creds version: 1 source: template bss-app-exist: false + nrm: # paramset: nrm version: 23.4 source: template + apps: + dbp: + storage: + basePath: "dbp-docs" + enabled: true + rb: + volumes: + outputs: + capacity: "10Gi" + storageClass: "csi-cinder-sc-retain" + identityProviders: + infra-keycloak: + credentials: + password: "${creds.get(\"keycloak-admin-cred\").password}" + username: "${creds.get(\"keycloak-admin-cred\").username}" + url: "https://infra-keycloak-infra-keycloak./realms/nrm-env-02" + usedFor: + - "implicit" + kafka: + password: "${creds.get(\"ocs-kafka-cred\").password}" + username: "${creds.get(\"ocs-kafka-cred\").username}" + monitoring: + grafana: + url: "https://grafana-platform-monitoring./" + namespace: "platform-monitoring" + postgresql: + password: "${creds.get(\"postgresql-cred\").password}" + ro: + address: "pg-patroni.postgresql:5432" + rw: + address: "pg-patroni.postgresql:5432" + username: "${creds.get(\"postgresql-cred\").username}" + prometheus-adapter: + install: "true" + storage: + url: "s3://${creds.get(\"s3-dbp-bucket-cred\").username}:${creds.get(\"s3-dbp-bucket-cred\").password}@s3.eu-central-1.amazonaws.com" e2eParameters: {} technicalConfigurationParameters: PARAM_2: "value-2" # paramset: bss-tech version: 25.1 source: template diff --git a/test_data/test_environments/cluster-01/env-02/Namespaces/core/namespace.yml b/test_data/test_environments/cluster-01/env-02/Namespaces/core/namespace.yml index 987a6d1a..89dc67b7 100644 --- a/test_data/test_environments/cluster-01/env-02/Namespaces/core/namespace.yml +++ b/test_data/test_environments/cluster-01/env-02/Namespaces/core/namespace.yml @@ -20,4 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_environments/cluster-01/env-02/Namespaces/oss/namespace.yml b/test_data/test_environments/cluster-01/env-02/Namespaces/oss/namespace.yml index 56a2fc24..e4ca944d 100644 --- a/test_data/test_environments/cluster-01/env-02/Namespaces/oss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-02/Namespaces/oss/namespace.yml @@ -20,5 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-02/Profiles/dev_billing_override.yml b/test_data/test_environments/cluster-01/env-02/Profiles/dev_billing_override.yml deleted file mode 100644 index 1d79f3e3..00000000 --- a/test_data/test_environments/cluster-01/env-02/Profiles/dev_billing_override.yml +++ /dev/null @@ -1,22 +0,0 @@ -# The contents of this file is generated from template artifact: deployment-configuration-env-templates. -# Contents will be overwritten by next generation. -# Please modify this contents only for development purposes or as workaround. -name: "dev_billing_override" -version: 0 -baseline: "dev" -description: "Override for dev-profile" -applications: - - name: "billing-app" - #version: "release-2023.3-8.4.0-20231103.225817-60-RELEASE" - #sd: "" - services: - - name: "billing-service" - parameters: - - name: "CPU_LIMIT" - value: "2" - - name: "REPLICAS" - value: "3" - - name: "CPU_REQUEST" - value: "150m" - - name: "MEMORY_LIMIT" - value: "336Mi" diff --git a/test_data/test_environments/cluster-01/env-02/composite_structure.yml b/test_data/test_environments/cluster-01/env-02/composite_structure.yml new file mode 100644 index 00000000..bed8a182 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-02/composite_structure.yml @@ -0,0 +1,9 @@ +name: "cluster_01_env_02-composite-structure" +baseline: + name: "env-02-core" + type: "namespace" +satellites: + - name: "env-02-bss" + type: "namespace" + - name: "env-02-oss" + type: "namespace" diff --git a/test_data/test_environments/cluster-01/env-02/tenant.yml b/test_data/test_environments/cluster-01/env-02/tenant.yml index 925818f5..896c21b6 100644 --- a/test_data/test_environments/cluster-01/env-02/tenant.yml +++ b/test_data/test_environments/cluster-01/env-02/tenant.yml @@ -16,4 +16,3 @@ globalE2EParameters: environmentParameters: {} deployParameters: ESCAPE_SEQUENCE: "true" - diff --git a/test_data/test_environments/cluster-01/env-03/Credentials/credentials.yml b/test_data/test_environments/cluster-01/env-03/Credentials/credentials.yml index aa58cb23..67140f8c 100644 --- a/test_data/test_environments/cluster-01/env-03/Credentials/credentials.yml +++ b/test_data/test_environments/cluster-01/env-03/Credentials/credentials.yml @@ -104,6 +104,11 @@ integration-cred: # namespace env-03-bss data: username: "envgeneNullValue" # FillMe password: "envgeneNullValue" # FillMe +keycloak-admin-cred: # namespace env-03-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe kms-cert: # namespace env-03-bss type: "secret" data: @@ -118,6 +123,21 @@ minio-cred: # cloud passport: cluster-01 version: 1.5 data: username: "minio-username" password: "minio-password" +ocs-kafka-cred: # namespace env-03-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +postgresql-cred: # namespace env-03-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +s3-dbp-bucket-cred: # namespace env-03-bss + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe service-integration-cred: # namespace env-03-bss type: "usernamePassword" data: diff --git a/test_data/test_environments/cluster-01/env-03/Namespaces/billing/namespace.yml b/test_data/test_environments/cluster-01/env-03/Namespaces/billing/namespace.yml index 85eba164..1fdda19d 100644 --- a/test_data/test_environments/cluster-01/env-03/Namespaces/billing/namespace.yml +++ b/test_data/test_environments/cluster-01/env-03/Namespaces/billing/namespace.yml @@ -10,11 +10,11 @@ labels: cleanInstallApprovalRequired: false mergeDeployParametersAndE2EParameters: false profile: - name: "dev_billing_override" + name: "rgrs_oc_profile" baseline: "dev" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" PARAM_2: "value-2" # paramset: billing version: 25.1 source: template PARAM_6: "value-6" # paramset: billing version: 25.1 source: template e2eParameters: {} @@ -22,5 +22,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-03/Namespaces/bss/namespace.yml b/test_data/test_environments/cluster-01/env-03/Namespaces/bss/namespace.yml index 1bfea3ee..2572917f 100644 --- a/test_data/test_environments/cluster-01/env-03/Namespaces/bss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-03/Namespaces/bss/namespace.yml @@ -13,12 +13,15 @@ profile: name: "dev_bss_override" baseline: "dev" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + DEPLOY_W_HELM: true # paramset: nrm version: 23.4 source: template + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" + ESCAPE_SEQUENCE: true # paramset: nrm version: 23.4 source: template KMS_CERT_IN_BASE64: "${envgen.creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template OVERRIDE_PARAMETER: "template-value" PARAM_2: "value-2" # paramset: bss version: 25.1 source: template PARAM_6: "value-6" # paramset: bss version: 25.1 source: template + STORAGE_USERNAME: "${creds.get(\"minio-cred\").username}" TEST_CREDS_GET_PASSWORD_PASSWORD: "${envgen.creds.get('creds-get-password-cred').password}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_PASSWORD_USERNAME: "${envgen.creds.get('creds-get-username-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_CREDS_GET_SECRET_PARAM: "${envgen.creds.get('creds-get-secret').secret}" # paramset: test-deploy-creds version: 1 source: template @@ -36,6 +39,43 @@ deployParameters: TEST_SHARED_CREDS: "${envgen.creds.get('integration-cred').username}" # paramset: test-deploy-creds version: 1 source: template TEST_SHARED_CREDS_ACTIVATOR: "${envgen.creds.get('service-integration-cred').password}" # paramset: test-deploy-creds version: 1 source: template bss-app-exist: false + nrm: # paramset: nrm version: 23.4 source: template + apps: + dbp: + storage: + basePath: "dbp-docs" + enabled: true + rb: + volumes: + outputs: + capacity: "10Gi" + storageClass: "csi-cinder-sc-retain" + identityProviders: + infra-keycloak: + credentials: + password: "${creds.get(\"keycloak-admin-cred\").password}" + username: "${creds.get(\"keycloak-admin-cred\").username}" + url: "https://infra-keycloak-infra-keycloak./realms/nrm-env-03" + usedFor: + - "implicit" + kafka: + password: "${creds.get(\"ocs-kafka-cred\").password}" + username: "${creds.get(\"ocs-kafka-cred\").username}" + monitoring: + grafana: + url: "https://grafana-platform-monitoring./" + namespace: "platform-monitoring" + postgresql: + password: "${creds.get(\"postgresql-cred\").password}" + ro: + address: "pg-patroni.postgresql:5432" + rw: + address: "pg-patroni.postgresql:5432" + username: "${creds.get(\"postgresql-cred\").username}" + prometheus-adapter: + install: "true" + storage: + url: "s3://${creds.get(\"s3-dbp-bucket-cred\").username}:${creds.get(\"s3-dbp-bucket-cred\").password}@s3.eu-central-1.amazonaws.com" e2eParameters: {} technicalConfigurationParameters: PARAM_2: "value-2" # paramset: bss-tech version: 25.1 source: template diff --git a/test_data/test_environments/cluster-01/env-03/Namespaces/core/namespace.yml b/test_data/test_environments/cluster-01/env-03/Namespaces/core/namespace.yml index 15bab12d..9553f959 100644 --- a/test_data/test_environments/cluster-01/env-03/Namespaces/core/namespace.yml +++ b/test_data/test_environments/cluster-01/env-03/Namespaces/core/namespace.yml @@ -20,4 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_environments/cluster-01/env-03/Namespaces/oss/namespace.yml b/test_data/test_environments/cluster-01/env-03/Namespaces/oss/namespace.yml index 75e70637..f488f1e5 100644 --- a/test_data/test_environments/cluster-01/env-03/Namespaces/oss/namespace.yml +++ b/test_data/test_environments/cluster-01/env-03/Namespaces/oss/namespace.yml @@ -20,5 +20,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-03/Profiles/dev_billing_override.yml b/test_data/test_environments/cluster-01/env-03/Profiles/dev_billing_override.yml deleted file mode 100644 index 1d79f3e3..00000000 --- a/test_data/test_environments/cluster-01/env-03/Profiles/dev_billing_override.yml +++ /dev/null @@ -1,22 +0,0 @@ -# The contents of this file is generated from template artifact: deployment-configuration-env-templates. -# Contents will be overwritten by next generation. -# Please modify this contents only for development purposes or as workaround. -name: "dev_billing_override" -version: 0 -baseline: "dev" -description: "Override for dev-profile" -applications: - - name: "billing-app" - #version: "release-2023.3-8.4.0-20231103.225817-60-RELEASE" - #sd: "" - services: - - name: "billing-service" - parameters: - - name: "CPU_LIMIT" - value: "2" - - name: "REPLICAS" - value: "3" - - name: "CPU_REQUEST" - value: "150m" - - name: "MEMORY_LIMIT" - value: "336Mi" diff --git a/test_data/test_environments/cluster-01/env-03/Profiles/rgrs_oc_profile.yml b/test_data/test_environments/cluster-01/env-03/Profiles/rgrs_oc_profile.yml new file mode 100644 index 00000000..6a34d597 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-03/Profiles/rgrs_oc_profile.yml @@ -0,0 +1,94 @@ +# The contents of this file is generated from template artifact: deployment-configuration-env-templates. +# Contents will be overwritten by next generation. +# Please modify this contents only for development purposes or as workaround. +name: "rgrs_oc_profile" +version: 0 +baseline: "dev" +description: "Custom profile for RGRS Openshift" +applications: + - name: "Access-Control" + version: "ANY" + services: + - name: "access-control" + parameters: + - name: "ACCESS_CONTROL_COMPOSITE_GATEWAY_MEMORY_LIMIT" + value: "256Mi" + - name: "Cloud-Core" + version: "ANY" + services: + - name: "facade-operator" + parameters: + - name: "FACADE_GATEWAY_MEMORY_LIMIT" + value: "256Mi" + - name: "frontend-gateway" + parameters: + - name: "INTERNAL_GW_MEMORY_LIMIT" + value: "475Mi" + - name: "PRIVATE_GW_MEMORY_LIMIT" + value: "475Mi" + - name: "PUBLIC_GW_MEMORY_LIMIT" + value: "475Mi" + - name: "Core-Extensions" + version: "ANY" + services: + - name: "tenant-self-service-frontend" + parameters: + - name: "NGINX_WORKER_PROCESSES" + value: "8" + - name: "CDN" + version: "ANY" + services: + - name: "cdn-frontend-service" + parameters: + - name: "NGINX_WORKER_PROCESSES" + value: "8" + - name: "platform" + version: "ANY" + services: + - name: "package-manager" + parameters: + - name: "CPU_LIMIT" + value: "2000m" + - name: "PM_HEAP_MAX_SIZE" + value: "2000m" + - name: "MEMORY_LIMIT" + value: "3Gi" + - name: "PM_BATCH_CAPACITY" + value: "40" + - name: "mediator-executor" + parameters: + - name: "CPU_LIMIT" + value: "2500m" + - name: "HPA_MAX_REPLICAS" + value: "8" + - name: "HPA_MIN_REPLICAS" + value: "4" + - name: "MEMORY_LIMIT" + value: "8Gi" + - name: "resource-inventory" + version: "ANY" + services: + - name: "validation-processor" + parameters: + - name: "CPU_LIMIT" + value: "250m" + - name: "MEMORY_LIMIT" + value: "512Mi" + - name: "resource-inventory.consolidated-inventory" + version: "ANY" + services: + - name: "device-library" + parameters: + - name: "CPU_LIMIT" + value: "2000m" + - name: "DL_HEAP_MAX_SIZE" + value: "2000m" + - name: "MEMORY_LIMIT" + value: "3Gi" + - name: "oss.orchestration.core" + version: "ANY" + services: + - name: "eso-manager" + parameters: + - name: "COMPOSITE_GATEWAY_MEMORY_LIMIT" + value: "256Mi" diff --git a/test_data/test_environments/cluster-01/env-03/composite_structure.yml b/test_data/test_environments/cluster-01/env-03/composite_structure.yml new file mode 100644 index 00000000..fed74dc4 --- /dev/null +++ b/test_data/test_environments/cluster-01/env-03/composite_structure.yml @@ -0,0 +1,9 @@ +name: "cluster_01_env_03-composite-structure" +baseline: + name: "env-03-core" + type: "namespace" +satellites: + - name: "env-03-bss" + type: "namespace" + - name: "env-03-oss" + type: "namespace" diff --git a/test_data/test_environments/cluster-01/env-03/tenant.yml b/test_data/test_environments/cluster-01/env-03/tenant.yml index 925818f5..896c21b6 100644 --- a/test_data/test_environments/cluster-01/env-03/tenant.yml +++ b/test_data/test_environments/cluster-01/env-03/tenant.yml @@ -16,4 +16,3 @@ globalE2EParameters: environmentParameters: {} deployParameters: ESCAPE_SEQUENCE: "true" - diff --git a/test_data/test_environments/cluster-01/env-04/Namespaces/billing/namespace.yml b/test_data/test_environments/cluster-01/env-04/Namespaces/billing/namespace.yml index 6ca7fadc..a930b593 100644 --- a/test_data/test_environments/cluster-01/env-04/Namespaces/billing/namespace.yml +++ b/test_data/test_environments/cluster-01/env-04/Namespaces/billing/namespace.yml @@ -13,12 +13,10 @@ profile: name: "" baseline: "dev" deployParameters: - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" e2eParameters: {} technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - - diff --git a/test_data/test_environments/cluster-01/env-04/cloud.yml b/test_data/test_environments/cluster-01/env-04/cloud.yml index 910505ee..cbacd692 100644 --- a/test_data/test_environments/cluster-01/env-04/cloud.yml +++ b/test_data/test_environments/cluster-01/env-04/cloud.yml @@ -76,4 +76,3 @@ deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] productionMode: false # cloud passport: cluster-01 version: 1.5 - diff --git a/test_data/test_environments/cluster-01/env-04/tenant.yml b/test_data/test_environments/cluster-01/env-04/tenant.yml index a5276705..c85259ab 100644 --- a/test_data/test_environments/cluster-01/env-04/tenant.yml +++ b/test_data/test_environments/cluster-01/env-04/tenant.yml @@ -15,4 +15,3 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: {} - diff --git a/test_data/test_environments/cluster01/env01/Inventory/env_definition.yml b/test_data/test_environments/cluster01/env01/Inventory/env_definition.yml index a5287595..0e61f254 100644 --- a/test_data/test_environments/cluster01/env01/Inventory/env_definition.yml +++ b/test_data/test_environments/cluster01/env01/Inventory/env_definition.yml @@ -1,10 +1,12 @@ inventory: environmentName: "test-solution-structure-case-01" - clusterUrl: "test-val.com" + clusterUrl: "https://api.qubership.org:6443/" tenantName: "test-tenant" deployer: "test-deployer" cloudName: "test-solution-structure" cloudPassport: "test-cloud-passport" + description: "" + owners: "" envTemplate: name: "test-solution-structure-template" additionalTemplateVariables: diff --git a/test_data/test_environments/cluster01/env01/Namespaces/app-core/namespace.yml b/test_data/test_environments/cluster01/env01/Namespaces/app-core/namespace.yml index 400bda5f..5aa4fa7b 100644 --- a/test_data/test_environments/cluster01/env01/Namespaces/app-core/namespace.yml +++ b/test_data/test_environments/cluster01/env01/Namespaces/app-core/namespace.yml @@ -15,8 +15,8 @@ profile: deployParameters: APP_CORE_NAMESPACE: "test-solution-structure-case-01-app-core" APP_NAMESPACE: "test-solution-structure-case-01-app" - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" TEST_PARAM_1: "test-solution-structure-case-01-app-core" # paramset: app-common version: n/a source: template TEST_PARAM_IF_ENDIF_2: "TEST_PARAM_IF_ENDIF_2_value" param-A: "apple" # paramset: paramset-A version: n/a source: template diff --git a/test_data/test_environments/cluster01/env01/Namespaces/app/namespace.yml b/test_data/test_environments/cluster01/env01/Namespaces/app/namespace.yml index 939182a4..bc9358c8 100644 --- a/test_data/test_environments/cluster01/env01/Namespaces/app/namespace.yml +++ b/test_data/test_environments/cluster01/env01/Namespaces/app/namespace.yml @@ -15,8 +15,8 @@ profile: deployParameters: APP_CORE_NAMESPACE: "test-solution-structure-case-01-app-core" APP_NAMESPACE: "test-solution-structure-case-01-app" - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" TEST_SS_1: "test-solution-structure-case-01-app-core" e2eParameters: {} technicalConfigurationParameters: @@ -26,4 +26,3 @@ deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] tenantName: "test-solution-structure-template" - diff --git a/test_data/test_environments/cluster01/env01/cloud.yml b/test_data/test_environments/cluster01/env01/cloud.yml index ae023540..2ed02be7 100644 --- a/test_data/test_environments/cluster01/env01/cloud.yml +++ b/test_data/test_environments/cluster01/env01/cloud.yml @@ -16,7 +16,7 @@ mergeDeployParametersAndE2EParameters: false maasConfig: credentialsId: "maas" enable: true - maasUrl: "http://maas.None" + maasUrl: "http://maas.qubership.org" maasInternalAddress: "http://maas.maas:8888" vaultConfig: credentialsId: "" @@ -26,11 +26,11 @@ dbaasConfigs: - credentialsId: "dbaas" enable: true apiUrl: "http://dbaas.dbaas:8888" - aggregatorUrl: "https://dbaas.None" + aggregatorUrl: "https://dbaas.qubership.org" consulConfig: tokenSecret: "consul-token" enabled: true - publicUrl: "https://consul.None" + publicUrl: "https://consul.qubership.org" internalUrl: "http://consul.consul:8888" deployParameters: CLOUD_DASHBOARD_URL: "https://dashboard.test-host.managed.tmp.cloud" # cloud passport: test-cloud-passport version: 1.5 diff --git a/test_data/test_environments/cluster01/env01/tenant.yml b/test_data/test_environments/cluster01/env01/tenant.yml index 88281d12..031498fc 100644 --- a/test_data/test_environments/cluster01/env01/tenant.yml +++ b/test_data/test_environments/cluster01/env01/tenant.yml @@ -15,4 +15,3 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: {} - diff --git a/test_data/test_environments/cluster01/env03/Namespaces/app/namespace.yml b/test_data/test_environments/cluster01/env03/Namespaces/app/namespace.yml index bfb80b12..a04af9b4 100644 --- a/test_data/test_environments/cluster01/env03/Namespaces/app/namespace.yml +++ b/test_data/test_environments/cluster01/env03/Namespaces/app/namespace.yml @@ -10,8 +10,8 @@ cleanInstallApprovalRequired: false mergeDeployParametersAndE2EParameters: false deployParameters: APP_NAMESPACE: "test-environment-app" # paramset: test-parameters-1 version: n/a source: template - ENVGENE_CONFIG_REF_NAME: "" - ENVGENE_CONFIG_TAG: "" + ENVGENE_CONFIG_REF_NAME: "No Ref Name" + ENVGENE_CONFIG_TAG: "No Ref tag" PARAMETERS_SD_VALUE: "PARAMETERS_SD_VALUE" # paramset: test-parameters-1 version: n/a source: template PARAMETERS_Variable_1: "Value 1" # paramset: test-parameters-1 version: n/a source: template PARAMETERS_Variable_2: "Value 2" # paramset: test-parameters-1 version: n/a source: template diff --git a/test_data/test_environments/cluster01/env03/cloud.yml b/test_data/test_environments/cluster01/env03/cloud.yml index e7671f90..40e534ba 100644 --- a/test_data/test_environments/cluster01/env03/cloud.yml +++ b/test_data/test_environments/cluster01/env03/cloud.yml @@ -46,4 +46,3 @@ deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] productionMode: false # cloud passport: test-cloud-passport version: 1.5 - diff --git a/test_data/test_environments/cluster01/env03/tenant.yml b/test_data/test_environments/cluster01/env03/tenant.yml index e6ee55f5..c14b64a1 100644 --- a/test_data/test_environments/cluster01/env03/tenant.yml +++ b/test_data/test_environments/cluster01/env03/tenant.yml @@ -15,4 +15,3 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: {} - diff --git a/test_data/test_environments/cluster01/env04/Credentials/credentials.yml b/test_data/test_environments/cluster01/env04/Credentials/credentials.yml new file mode 100644 index 00000000..c3908c30 --- /dev/null +++ b/test_data/test_environments/cluster01/env04/Credentials/credentials.yml @@ -0,0 +1,18 @@ +admin-token: # cloud _test_environment + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +consul-token: # cloud _test_environment + type: "secret" + data: + secret: "envgeneNullValue" # FillMe +dbaas: # cloud _test_environment + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe +maas: # cloud _test_environment + type: "usernamePassword" + data: + username: "envgeneNullValue" # FillMe + password: "envgeneNullValue" # FillMe diff --git a/test_data/test_environments/cluster01/env04/cloud.yml b/test_data/test_environments/cluster01/env04/cloud.yml index 4478d578..384d318c 100644 --- a/test_data/test_environments/cluster01/env04/cloud.yml +++ b/test_data/test_environments/cluster01/env04/cloud.yml @@ -39,4 +39,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_environments/cluster01/env04/tenant.yml b/test_data/test_environments/cluster01/env04/tenant.yml index 140b9559..9193febb 100644 --- a/test_data/test_environments/cluster01/env04/tenant.yml +++ b/test_data/test_environments/cluster01/env04/tenant.yml @@ -15,4 +15,3 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: {} - diff --git a/test_data/test_environments/cluster01/single_sd/Inventory/solution-descriptor/sd.yml b/test_data/test_environments/cluster01/single_sd/Inventory/solution-descriptor/sd.yml deleted file mode 100644 index e94e2ff1..00000000 --- a/test_data/test_environments/cluster01/single_sd/Inventory/solution-descriptor/sd.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '2.1' -type: solutionDeploy -deployMode: composite -applications: - - version: MONITORING:0.64.1 - deployPostfix: platform-monitoring - - version: postgres:1.32.6 - deployPostfix: postgresql - - version: postgres-services:1.32.6 - deployPostfix: postgresql - - version: postgres:1.32.6 - deployPostfix: postgresql-dbaas diff --git a/test_data/test_templates/env_templates/composite-dev.yaml b/test_data/test_templates/env_templates/composite-dev.yaml index 2f4b6448..329b72dd 100644 --- a/test_data/test_templates/env_templates/composite-dev.yaml +++ b/test_data/test_templates/env_templates/composite-dev.yaml @@ -9,5 +9,6 @@ namespaces: deploy_postfix: "bss" - template_path: "{{ templates_dir }}/env_templates/composite/namespaces/oss.yml.j2" deploy_postfix: "oss" - - template_path: "{{ templates_dir }}/env_templates/composite/namespaces/billing.yaml.j2" + - template_path: "{{ templates_dir }}/env_templates/composite/namespaces/billing_not_deploy_postfix.yaml.j2" deploy_postfix: "billing" +composite_structure: "{{ templates_dir }}/env_templates/composite/composite_structure.yml.j2" diff --git a/test_data/test_templates/env_templates/composite-prod.yaml b/test_data/test_templates/env_templates/composite-prod.yaml index 4982c986..89fa28ca 100644 --- a/test_data/test_templates/env_templates/composite-prod.yaml +++ b/test_data/test_templates/env_templates/composite-prod.yaml @@ -23,7 +23,7 @@ namespaces: profile: name: prod_oss_override baseline: prod - - template_path: "{{ templates_dir }}/env_templates/composite/namespaces/billing.yaml.j2" + - template_path: "{{ templates_dir }}/env_templates/composite/namespaces/billing_not_deploy_postfix.yaml.j2" deploy_postfix: "billing" template_override: profile: diff --git a/test_data/test_templates/env_templates/composite/composite_structure.yml.j2 b/test_data/test_templates/env_templates/composite/composite_structure.yml.j2 new file mode 100644 index 00000000..78a2085a --- /dev/null +++ b/test_data/test_templates/env_templates/composite/composite_structure.yml.j2 @@ -0,0 +1,9 @@ +name: "{{ current_env.cloudNameWithCluster }}-composite-structure" +baseline: + name: "{{ current_env.name }}-core" + type: "namespace" +satellites: + - name: "{{ current_env.name }}-bss" + type: "namespace" + - name: "{{ current_env.name }}-oss" + type: "namespace" diff --git a/test_data/test_templates/env_templates/composite/namespaces/billing.yaml.j2 b/test_data/test_templates/env_templates/composite/namespaces/billing_not_deploy_postfix.yaml.j2 similarity index 96% rename from test_data/test_templates/env_templates/composite/namespaces/billing.yaml.j2 rename to test_data/test_templates/env_templates/composite/namespaces/billing_not_deploy_postfix.yaml.j2 index ba2a6f97..250f29e8 100644 --- a/test_data/test_templates/env_templates/composite/namespaces/billing.yaml.j2 +++ b/test_data/test_templates/env_templates/composite/namespaces/billing_not_deploy_postfix.yaml.j2 @@ -8,7 +8,7 @@ labels: cleanInstallApprovalRequired: {{ current_env.additionalTemplateVariables.cleanInstallApprovalRequired | default('false') }} mergeDeployParametersAndE2EParameters: false profile: - name: dev_billing_override + name: rgrs_oc_profile baseline: dev deployParameters: ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" @@ -19,4 +19,3 @@ deployParameterSets: - "billing" e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_templates/env_templates/composite/namespaces/bss.yml.j2 b/test_data/test_templates/env_templates/composite/namespaces/bss.yml.j2 index 8960d7a8..b01c869f 100644 --- a/test_data/test_templates/env_templates/composite/namespaces/bss.yml.j2 +++ b/test_data/test_templates/env_templates/composite/namespaces/bss.yml.j2 @@ -1,4 +1,6 @@ --- +{%- if current_env.additionalTemplateVariables.use_env_prefix is defined %} +{%- endif %} name: "{{current_env.name}}-bss" credentialsId: "" isServerSideMerge: false @@ -14,6 +16,7 @@ deployParameters: ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" OVERRIDE_PARAMETER: "template-value" + STORAGE_USERNAME: "{{current_env.cloud_passport.storage.STORAGE_USERNAME}}" {% if 'bss-app' in current_env.solution_structure %} bss-app-exist: true {% else %} @@ -29,6 +32,7 @@ technicalConfigurationParameters: {} deployParameterSets: - "bss" - "test-deploy-creds" +- "nrm" e2eParameterSets: [] technicalConfigurationParameterSets: - "bss-tech" diff --git a/test_data/test_templates/env_templates/simple/billing.yml.j2 b/test_data/test_templates/env_templates/simple/billing.yml.j2 index c3b5dc4e..1a7afadb 100644 --- a/test_data/test_templates/env_templates/simple/billing.yml.j2 +++ b/test_data/test_templates/env_templates/simple/billing.yml.j2 @@ -18,4 +18,3 @@ technicalConfigurationParameters: {} deployParameterSets: [] e2eParameterSets: [] technicalConfigurationParameterSets: [] - diff --git a/test_data/test_templates/env_templates/test-template-1/tenant.yml.j2 b/test_data/test_templates/env_templates/test-template-1/tenant.yml.j2 index 11cfdf42..96f19549 100644 --- a/test_data/test_templates/env_templates/test-template-1/tenant.yml.j2 +++ b/test_data/test_templates/env_templates/test-template-1/tenant.yml.j2 @@ -1,4 +1,4 @@ -name: "{{current_env.tenant}}" +name: "{{_tenant}}" registryName: "default" description: "{{current_env.description}}" owners: "{{current_env.owners}}" diff --git a/test_data/test_templates/parameters/nrm.yml.j2 b/test_data/test_templates/parameters/nrm.yml.j2 new file mode 100644 index 00000000..9109a59e --- /dev/null +++ b/test_data/test_templates/parameters/nrm.yml.j2 @@ -0,0 +1,44 @@ +version: 23.4 +name: nrm +parameters: + DEPLOY_W_HELM: true + ESCAPE_SEQUENCE: true + nrm: + postgresql: + username: ${creds.get("postgresql-cred").username} + password: ${creds.get("postgresql-cred").password} + rw: + address: "pg-patroni.{{ current_env.additionalTemplateVariables.namespace_postgresql | default('postgresql') }}:5432" + ro: + address: "pg-patroni.{{ current_env.additionalTemplateVariables.namespace_postgresql | default('postgresql') }}:5432" + storage: + url: s3://${creds.get("s3-dbp-bucket-cred").username}:${creds.get("s3-dbp-bucket-cred").password}@{{ current_env.additionalTemplateVariables.s3_endpoint | default('s3.eu-central-1.amazonaws.com') }} + kafka: + username: ${creds.get("ocs-kafka-cred").username} + password: ${creds.get("ocs-kafka-cred").password} + apps: + rb: + volumes: + outputs: + storageClass: "{{ current_env.additionalTemplateVariables.storage_class_rwx | default('csi-cinder-sc-retain') }}" + capacity: 10Gi + dbp: + storage: + enabled: true + basePath: dbp-docs + identityProviders: + infra-keycloak: + url: "https://infra-keycloak-{{ current_env.additionalTemplateVariables.namespace_infra_keycloak | default('infra-keycloak') }}.{{ current_env.cluster.cloud_public_url }}/realms/nrm-{{ current_env.environmentName }}" + credentials: + username: "${creds.get(\"keycloak-admin-cred\").username}" + password: "${creds.get(\"keycloak-admin-cred\").password}" + usedFor: + - implicit + monitoring: + namespace: "{{ current_env.additionalTemplateVariables.namespace_monitoring | default('platform-monitoring') }}" + grafana: + url: https://grafana-{{ current_env.additionalTemplateVariables.namespace_monitoring | default('platform-monitoring') }}.{{ current_env.cluster.cloud_public_url }}/ + prometheus-adapter: + install: "true" + + diff --git a/test_data/test_templates/resource_profiles/rgrs_oc_profile.yml.j2 b/test_data/test_templates/resource_profiles/rgrs_oc_profile.yml.j2 new file mode 100644 index 00000000..e5d9657a --- /dev/null +++ b/test_data/test_templates/resource_profiles/rgrs_oc_profile.yml.j2 @@ -0,0 +1,102 @@ +--- +name: "rgrs_oc_profile" +{% if current_env.cloud_passport.custom.BASELINE == 'dev' %} +baseline: "dev" +{% elif current_env.cloud_passport.custom.BASELINE == 'prod' %} +baseline: "prod" +{% elif current_env.cloud_passport.custom.BASELINE == 'prod-nonha' %} +baseline: "prod-nonha" +{% elif current_env.cloud_passport.custom.BASELINE == 'dev-ha' %} +baseline: "dev-ha" +{% else %} +baseline: "dev" +{% endif %} +version: 0 +description: "Custom profile for RGRS Openshift" +applications: +- name: "Access-Control" + version: "ANY" + services: + - name: "access-control" + parameters: + - name: "ACCESS_CONTROL_COMPOSITE_GATEWAY_MEMORY_LIMIT" + value: "256Mi" +- name: "Cloud-Core" + version: "ANY" + services: + - name: "facade-operator" + parameters: + - name: "FACADE_GATEWAY_MEMORY_LIMIT" + value: "256Mi" + - name: "frontend-gateway" + parameters: + - name: "INTERNAL_GW_MEMORY_LIMIT" + value: "475Mi" + - name: "PRIVATE_GW_MEMORY_LIMIT" + value: "475Mi" + - name: "PUBLIC_GW_MEMORY_LIMIT" + value: "475Mi" +- name: "Core-Extensions" + version: "ANY" + services: + - name: "tenant-self-service-frontend" + parameters: + - name: "NGINX_WORKER_PROCESSES" + value: "8" +- name: "CDN" + version: "ANY" + services: + - name: "cdn-frontend-service" + parameters: + - name: "NGINX_WORKER_PROCESSES" + value: "8" +- name: "platform" + version: "ANY" + services: + - name: "package-manager" + parameters: + - name: "CPU_LIMIT" + value: "2000m" + - name: "PM_HEAP_MAX_SIZE" + value: "2000m" + - name: "MEMORY_LIMIT" + value: "3Gi" + - name: "PM_BATCH_CAPACITY" + value: "40" + - name: "mediator-executor" + parameters: + - name: "CPU_LIMIT" + value: "2500m" + - name: "HPA_MAX_REPLICAS" + value: "8" + - name: "HPA_MIN_REPLICAS" + value: "4" + - name: "MEMORY_LIMIT" + value: "8Gi" +- name: "resource-inventory" + version: "ANY" + services: + - name: "validation-processor" + parameters: + - name: "CPU_LIMIT" + value: "250m" + - name: "MEMORY_LIMIT" + value: "512Mi" +- name: "resource-inventory.consolidated-inventory" + version: "ANY" + services: + - name: "device-library" + parameters: + - name: "CPU_LIMIT" + value: "2000m" + - name: "DL_HEAP_MAX_SIZE" + value: "2000m" + - name: "MEMORY_LIMIT" + value: "3Gi" +- name: "oss.orchestration.core" + version: "ANY" + services: + - name: "eso-manager" + parameters: + - name: "COMPOSITE_GATEWAY_MEMORY_LIMIT" + value: "256Mi"