diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 100755 index 00000000..56edf89d --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1,12 @@ +#!/bin/sh +msg_file="$1" +first_line=$(head -n 1 "$msg_file") + +# Allowed prefixes (extend if needed) +case "$first_line" in + fix:*|feat:*|chore:*|docs:*|refactor:*|test:*|ci:*) + exit 0 ;; # ok +esac + +# Prepend "fix:" if no prefix +sed -i "1s|^|fix: |" "$msg_file" diff --git a/.gitignore b/.gitignore index 7a77e7ff..44a0ad14 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ venv __pycache__ .vscode .DS_Store +pyrightconfig.json +/qwerty diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..4270b994 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +compose := sudo docker compose -f devtools/docker-compose.yml + +build-%: + $(compose) build $* + +up-%: + $(compose) up -d $* + @if [ -f devtools/$*/up.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/up.sh; fi + +bash-%: + $(compose) exec $* bash + +down: + $(compose) down + +stop-%: + $(compose) stop $* + +rm-%: + $(compose) rm $* + +run-%: + @if [ -f devtools/$*/run.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/run.sh; \ + else echo "No run script for $*"; fi + +edit: + vim $(abspath $(lastword $(MAKEFILE_LIST))) diff --git a/build_pipegene/scripts/env_build_jobs.py b/build_pipegene/scripts/env_build_jobs.py index a6c3fea3..9bb127bd 100644 --- a/build_pipegene/scripts/env_build_jobs.py +++ b/build_pipegene/scripts/env_build_jobs.py @@ -21,6 +21,8 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full else: script.append("export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')") + script.append("python /build_env/scripts/build_env/validate_bgd.py") + script.append("python /build_env/scripts/build_env/filter_namespaces.py") script.extend([ 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' @@ -41,6 +43,7 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full env_build_vars = { "ENV_NAME": full_env, + "FULL_ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": enviroment_name, "ENV_TEMPLATE_VERSION": env_template_version, @@ -138,6 +141,7 @@ def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, de git_commit_job = job_instance(params=git_commit_params, vars=git_commit_vars) git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/" + f"{full_env}") git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/git_envs") + git_commit_job.artifacts.add_paths('${CI_PROJECT_DIR}/sboms') git_commit_job.artifacts.when = WhenStatement.ALWAYS if (credential_rotation_job is not None): git_commit_job.add_needs(credential_rotation_job) diff --git a/build_pipegene/scripts/gitlab_ci.py b/build_pipegene/scripts/gitlab_ci.py index 47a76ca4..12aeb6d8 100644 --- a/build_pipegene/scripts/gitlab_ci.py +++ b/build_pipegene/scripts/gitlab_ci.py @@ -22,7 +22,7 @@ def build_pipeline(params: dict): # if we are in template testing during template build tags=params['GITLAB_RUNNER_TAG_NAME'] - + if params['IS_TEMPLATE_TEST']: logger.info("We are generating jobs in template test mode.") templates_dir = f"{project_dir}/templates/env_templates" @@ -117,21 +117,23 @@ def build_pipeline(params: dict): else: logger.info(f'Preparing of generate_effective_set job for {cluster_name}/{environment_name} is skipped.') - ## git_commit job - jobs_requiring_git_commit = ("env_build_job", "generate_effective_set_job", "env_inventory_generation_job", "credential_rotation_job") - if any(job in jobs_map for job in jobs_requiring_git_commit) and not params['IS_TEMPLATE_TEST']: - jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, env, environment_name, cluster_name, params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job) - else: - logger.info(f'Preparing of git commit job for {env} is skipped.') + jobs_requiring_git_commit = ["env_build_job", "generate_effective_set_job", "env_inventory_generation_job", "credential_rotation_job"] plugin_params = params plugin_params['jobs_map'] = jobs_map plugin_params['job_sequence'] = job_sequence + plugin_params['jobs_requiring_git_commit'] = jobs_requiring_git_commit plugin_params['env_name'] = environment_name plugin_params['cluster_name'] = cluster_name plugin_params['full_env'] = env per_env_plugin_engine.run(params=plugin_params, pipeline=pipeline, pipeline_helper=pipeline_helper) + ## git_commit job + if any(job in jobs_map for job in plugin_params['jobs_requiring_git_commit']) and not params['IS_TEMPLATE_TEST']: + jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, env, environment_name, cluster_name, params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job) + else: + logger.info(f'Preparing of git commit job for {env} is skipped.') + for job in job_sequence: if job not in jobs_map.keys(): continue diff --git a/build_pipegene/scripts/pipeline_parameters.py b/build_pipegene/scripts/pipeline_parameters.py index 510165d5..af1f245a 100644 --- a/build_pipegene/scripts/pipeline_parameters.py +++ b/build_pipegene/scripts/pipeline_parameters.py @@ -28,8 +28,9 @@ def get_pipeline_parameters() -> dict: }, 'CRED_ROTATION_PAYLOAD': getenv("CRED_ROTATION_PAYLOAD", ""), 'CRED_ROTATION_FORCE': getenv("CRED_ROTATION_FORCE", ""), + 'NS_BUILD_FILTER': getenv("NS_BUILD_FILTER", ""), 'GITLAB_RUNNER_TAG_NAME' : getenv("GITLAB_RUNNER_TAG_NAME", ""), - 'RUNNER_SCRIPT_TIMEOUT' : getenv("RUNNER_SCRIPT_TIMEOUT") or "10m" + 'RUNNER_SCRIPT_TIMEOUT' : getenv("RUNNER_SCRIPT_TIMEOUT") or "10m", } class PipelineParametersHandler: diff --git a/devtools/docker-compose.yml b/devtools/docker-compose.yml new file mode 100644 index 00000000..889fd639 --- /dev/null +++ b/devtools/docker-compose.yml @@ -0,0 +1,19 @@ +x-base-env: &base-env + CI_PROJECT_DIR: /workspace + IS_LOCAL_DEV_TEST_ENVGENE: "true" + +x-base-service: &base-service + volumes: + - ../:/workspace + working_dir: /workspace + entrypoint: sleep infinity + +services: + tests: + <<: *base-service + environment: + <<: *base-env + build: + dockerfile: devtools/tests/Dockerfile + context: ../ + diff --git a/devtools/readme.md b/devtools/readme.md new file mode 100644 index 00000000..dde59fab --- /dev/null +++ b/devtools/readme.md @@ -0,0 +1,43 @@ +# Guide for running things locally + +## Prerequisites + +1. You have installed `WSL` and have `docker` in it. + +## Setup + +1. Check `docker-compose.yml` file to find the service for general unit tests, + or job that you need to test and run +1. Run `make build-%` and `make up-%` where `%` is name of the service to get + container running for service that you want. And run `make run-%` to actually + run the code. + +## Usage + +Format: + +```sh +make - +``` + +Where `service` matches the service name from docker-compose.yml. + +| Command | What it does | Example | +| ------- | ----------------------------------------- | -------------------- | +| `build` | Build service image | `make build-tests` | +| `up` | Start service (runs `up.sh` if present) | `make up-tests` | +| `bash` | Open shell in running service | `make bash-tests` | +| `down` | Stop all services | `make down` | +| `stop` | Stop one service | `make stop-tests` | +| `rm` | Remove service container | `make rm-tests` | +| `run` | Run service’s `run.sh` script (if exists) | `make run-tests` | +| `edit` | Edit Makefile in Vim | `make edit` | + +**Notes:** + +* `up.sh` / `run.sh` are optional, per-service scripts. +* If service is configured correctly and has the code mounted + into it from repos, changes made in code in repos, should reflect immediately + in container, so after code changes you would need to just run `make run-%`. + If there are changes in docker-compose - run `make up-%` again and if there + are changes in `Dockerfile` itself used for service - run `make build-%` diff --git a/devtools/tests/Dockerfile b/devtools/tests/Dockerfile new file mode 100644 index 00000000..4ae99f8c --- /dev/null +++ b/devtools/tests/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12-slim-bookworm + +RUN useradd -m appuser +USER appuser + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + curl=7.88.1-10+deb12u8 \ + && rm -rf /var/lib/apt/lists/* + +# sops +RUN curl -LO https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 && \ + mv sops-v3.9.0.linux.amd64 /usr/local/bin/sops && \ + chmod +x /usr/local/bin/sops + +# Optional pip.conf / sources.list +COPY dependencies/pip.conf /etc/pip.conf +COPY dependencies/sources.list /etc/apt/sources.list + +# Copy dependencies and install them +COPY dependencies/tests_requirements.txt /tmp/ +RUN pip install --no-cache-dir "uv>=0.9.5" && uv pip install --system --no-cache-dir -r /tmp/tests_requirements.txt + diff --git a/devtools/tests/run.sh b/devtools/tests/run.sh new file mode 100755 index 00000000..5a12db40 --- /dev/null +++ b/devtools/tests/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euxo + +cd "$CI_PROJECT_DIR" + +# Run tests +cd python/envgene/envgenehelper +pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../../junit.xml +cd ../../.. +mv junit.xml junit_envgenehelper.xml + +cd scripts/build_env +pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../junit.xml +cd ../.. +mv junit.xml junit_build_env.xml + +# Merge results +python -m junitparser merge junit_build_env.xml junit_envgenehelper.xml junit.xml diff --git a/devtools/tests/up.sh b/devtools/tests/up.sh new file mode 100755 index 00000000..f57ba0a2 --- /dev/null +++ b/devtools/tests/up.sh @@ -0,0 +1,3 @@ +#!/bin/bash +chmod +x /workspace/python/build_modules.sh +/workspace/python/build_modules.sh diff --git a/docs/instance-pipeline-parameters.md b/docs/instance-pipeline-parameters.md index 91656f3e..2b828f5f 100644 --- a/docs/instance-pipeline-parameters.md +++ b/docs/instance-pipeline-parameters.md @@ -26,6 +26,9 @@ - [`CRED_ROTATION_PAYLOAD`](#cred_rotation_payload) - [Affected Parameters and Troubleshooting](#affected-parameters-and-troubleshooting) - [`CRED_ROTATION_FORCE`](#cred_rotation_force) + - [`SD_REPO_MERGE_MODE`](#sd_repo_merge_mode) + - [`NS_BUILD_FILTER`](#ns_build_filter) + - [`GITHUB_PIPELINE_API_INPUT`](#github_pipeline_api_input) - [`GH_ADDITIONAL_PARAMS`](#gh_additional_params) - [Deprecated Parameters](#deprecated-parameters) - [`SD_DELTA`](#sd_delta) diff --git a/env-builder/main.yaml b/env-builder/main.yaml index 8ecacb11..169bd1bb 100644 --- a/env-builder/main.yaml +++ b/env-builder/main.yaml @@ -118,6 +118,17 @@ loop_control: loop_var: namespace + - name: Gen BG Domain + ansible.builtin.include_role: + name: generate_bgd + vars: + _env: "{{ env }}" + _tenant: "{{ tenant }}" + _cloud: "{{ current_env.cloud }}" + _current_env_dir: "{{ current_env_dir }}" + _template: "{{current_env_template.bg_domain}}" + when: current_env_template.bg_domain is defined + - name: Gen resource profiles ansible.builtin.include_role: name: generate_profiles @@ -263,4 +274,3 @@ not use_external_defs and ((appdef_templates.files is defined and appdef_templates.files | length > 0) or (regdef_templates.files is defined and regdef_templates.files | length > 0)) - diff --git a/env-builder/roles/generate_bgd/tasks/main.yml b/env-builder/roles/generate_bgd/tasks/main.yml new file mode 100644 index 00000000..aa2bb8b0 --- /dev/null +++ b/env-builder/roles/generate_bgd/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Generate BG Domain yaml {{ _bg_domain }} + ansible.builtin.blockinfile: + path: "{{ current_env_dir }}/bg_domain.yml" + block: "{{ lookup('template', _template) }}" + create: yes diff --git a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml index e6cb1575..0a748e2e 100644 --- a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml +++ b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml @@ -69,9 +69,9 @@ env: DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator" #DOCKER_IMAGE_TAGS - DOCKER_IMAGE_TAG_PIPEGENE: "1.6.5" - DOCKER_IMAGE_TAG_ENVGENE: "1.6.5" - DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.6.5" + DOCKER_IMAGE_TAG_PIPEGENE: "feature_ns-filter-support_20251028-053643" + DOCKER_IMAGE_TAG_ENVGENE: "feature_ns-filter-support_20251028-053639" + DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "feature_ns-filter-support_20251028-053736" jobs: process_environment_variables: diff --git a/python/build_modules.sh b/python/build_modules.sh index 9b9200f8..b01c89b1 100755 --- a/python/build_modules.sh +++ b/python/build_modules.sh @@ -9,10 +9,15 @@ install_and_clean() { local base_path="${3:-$SCRIPTPATH}" # optional third argument to function echo "Installing $base_path/$path" - if [ "$IS_LOCAL_DEV_TEST_ENVGENE" = "true" ]; then - uv pip install --editable "$base_path/$path" + if [ "$USE_UV" = "true" ]; then + echo "USE_UV is true, using uv" + if ! command -v uv &>/dev/null; then + echo "uv not found, installing..." + pip install uv + fi + uv pip install --editable "$base_path/$path" else - pip install "$base_path/$path" + pip install "$base_path/$path" fi echo "Removing build trash..." diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index ec53a0e9..d255e68a 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -1,6 +1,11 @@ +from dataclasses import dataclass, field import pathlib import re from os import getenv +from pathlib import Path +from typing import overload + +from ruyaml import CommentedMap from .collections_helper import merge_lists from .yaml_helper import findYamls, openYaml, yaml, writeYamlToFile, store_value_to_yaml, validate_yaml_by_scheme_or_fail @@ -36,6 +41,11 @@ def find_env_instances_dir(env_name, instances_dir) : logger.error(f"Directory for {env_name} is not found in {instances_dir}") raise ReferenceError(f"Can't find directory for {env_name}") +@overload +def getenv_and_log(name: str) -> None | str: ... +@overload +def getenv_and_log(name: str, default: str) -> str: ... + def getenv_and_log(name, *args, **kwargs): var = getenv(name, *args, **kwargs) logger.info(f"{name}: {var}") @@ -45,11 +55,19 @@ def getenv_with_error(var_name): var = getenv(var_name) if not var: raise ValueError(f'Required value was not given and is not set in environment as {var_name}') + logger.debug(f"{var_name}: {var}") return var def get_env_instances_dir(environment_name, cluster_name, instances_dir): return f"{instances_dir}/{cluster_name}/{environment_name}" +def get_current_env_dir_with_env_vars() -> Path: + instances_dir = getenv_with_error('CI_PROJECT_DIR') + env_name = getenv_with_error('FULL_ENV_NAME') + env_dir_path = Path(f"{instances_dir}/environments/{env_name}") + logger.debug(env_dir_path) + return env_dir_path + def check_environment_is_valid_or_fail(environment_name, cluster_name, instances_dir, skip_env_definition_check=False, validate_env_definition_by_schema=False, schemas_dir=""): env_dir = get_env_instances_dir(environment_name, cluster_name, instances_dir) # check that environment directory exists @@ -294,3 +312,53 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): else: return "" +@dataclass +class Namespace: + path: Path + name: str = field(init=False) + definition_path: Path = field(init=False) + + def __post_init__(self): + self.definition_path = self.path.joinpath('namespace.yml') + self.name = openYaml(self.definition_path)['name'] + +def get_namespaces_path(env_dir: Path | None = None) -> Path: + env_dir = env_dir or get_current_env_dir_with_env_vars() + namespaces_path = env_dir.joinpath('Namespaces') + logger.debug(namespaces_path) + return namespaces_path + +def get_namespaces(env_dir: Path | None = None) -> list[Namespace]: + namespaces_path = get_namespaces_path(env_dir) + if not check_dir_exists(str(namespaces_path)): + return [] + namespace_paths = [p for p in namespaces_path.iterdir() if p.is_dir()] + namespaces = [Namespace(path=p) for p in namespace_paths] + logger.debug(namespaces) + return namespaces + +def get_bgd_path() -> Path: + env_dir = get_current_env_dir_with_env_vars() + bgd_path = env_dir.joinpath('bg_domain.yml') + logger.debug(bgd_path) + return bgd_path + +def get_bgd_object() -> CommentedMap: + bgd_path = get_bgd_path() + bgd_object = openYaml(bgd_path, allow_default=True) + logger.debug(bgd_object) + return bgd_object + +def make_relative_to_base_path(base: Path, target: Path) -> Path: + base = base.resolve() + target = target.resolve() + return target.relative_to(base) + +def make_relative_to_ci_project_dir(path: Path) -> Path: + ci_project_dir = Path(getenv_with_error("CI_PROJECT_DIR")) + return make_relative_to_base_path(ci_project_dir, path) + +def make_relative_to_env_dir(path: Path) -> Path: + env_dir = get_current_env_dir_with_env_vars() + return make_relative_to_base_path(env_dir, path) + diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index 8ebd1765..8f0fdf20 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -95,6 +95,10 @@ def openFileAsString(filePath): def deleteFile(filePath): os.remove(filePath) +def deleteFileIfExists(filePath): + if check_file_exists(filePath): + os.remove(filePath) + def writeToFile(filePath, contents): os.makedirs(os.path.dirname(filePath), exist_ok=True) with open(filePath, 'w+') as f: diff --git a/python/envgene/envgenehelper/yaml_helper.py b/python/envgene/envgenehelper/yaml_helper.py index 58893ecd..ef8315f5 100644 --- a/python/envgene/envgenehelper/yaml_helper.py +++ b/python/envgene/envgenehelper/yaml_helper.py @@ -40,7 +40,7 @@ def openYaml(filePath, safe_load=False, default_yaml: Callable=get_empty_yaml, a resultYaml = readYaml(f.read(), safe_load, context=f"File: {filePath}") return resultYaml -def readYaml(text, safe_load=False, context=None): +def readYaml(text, safe_load=False, context=None) -> CommentedMap: if text is None: resultYaml = None elif safe_load: diff --git a/scripts/build_env/create_credentials.py b/scripts/build_env/create_credentials.py index ab9c5bf3..10b5eb3c 100644 --- a/scripts/build_env/create_credentials.py +++ b/scripts/build_env/create_credentials.py @@ -73,6 +73,12 @@ def getCloudCreds(cloudContent, tenantName, cloudName): return creds +def get_bg_domain_creds(content, name): + creds = [] + bg_domain_comment = f"bg domain {name}" + checkCredAndAppend(content["controllerNamespace"]["credentials"], creds, CRED_TYPE_SECRET, bg_domain_comment) + return creds + def getNamespaceCreds(namespaceContent, tenantName, cloudName, namespaceName): creds = [] namespaceComment = f"namespace {namespaceName}" @@ -203,6 +209,17 @@ def create_credentials(envDir, envInstancesDir, instancesDir) : mergeResult = mergeCreds(getCloudCreds(cloudYaml, tenantName, cloudName), resultingCreds) logger.info(f'{mergeResult["countAdded"]} creds added from cloud {cloudFileName}') resultingCreds = mergeResult["mergedCreds"] + #bgd object + bgdFileName = envDir+"/bg_domain.yml" + logger.info(f"Processing bg domain") + if check_file_exists(bgdFileName): + bgd_yaml = openYaml(bgdFileName) + bgd_name = bgd_yaml["name"] + mergeResult = mergeCreds(get_bg_domain_creds(bgd_yaml, bgd_name), resultingCreds) + logger.info(f'{mergeResult["countAdded"]} creds added from bg domain {bgdFileName}') + resultingCreds = mergeResult["mergedCreds"] + else: + logger.info("Bg domain doesn't exist") # iterate through cloud applications and create cred definitions applications = findAllYamlsInDir(f"{envDir}/Applications") for appPath in applications : diff --git a/scripts/build_env/filter_namespaces.py b/scripts/build_env/filter_namespaces.py new file mode 100644 index 00000000..f39032d6 --- /dev/null +++ b/scripts/build_env/filter_namespaces.py @@ -0,0 +1,71 @@ +from pathlib import Path +import shutil + +from envgenehelper.business_helper import Namespace, get_bgd_object, get_namespaces, get_namespaces_path, getenv_and_log, getenv_with_error +from envgenehelper import logger + +def filter_namespaces(namespaces: list[str], filter: str, bgd_object: dict) -> list[str]: + if not filter: + return namespaces + is_exclusion = filter.startswith("!") + if is_exclusion: filter = filter[1:] + selectors = [s.strip() for s in filter.split(",") if s.strip()] + resolved_filter = [] + + for sel in selectors: + if not sel.startswith("@"): + resolved_filter.append(sel) + continue + alias = sel[1:] + 'Namespace' + if alias not in bgd_object: + raise ValueError(f"Unknown alias in NS_BUILD_FILTER: {sel}, can't find {alias} in BGD object") + name = bgd_object[alias]["name"] + + resolved_filter.append(name) + if is_exclusion: + filtered_namespaces = [ns for ns in namespaces if ns not in resolved_filter] + else: + filtered_namespaces = [ns for ns in namespaces if ns in resolved_filter] + return filtered_namespaces + +def apply_ns_build_filter(): + filter = getenv_and_log('NS_BUILD_FILTER', default='') + logger.info(f"Filtering namespaces with NS_BUILD_FILTER: {filter}") + base_dir = getenv_with_error("CI_PROJECT_DIR") + + source_namespaces = get_namespaces(Path(f'{base_dir}/build_env/tmp/initial_namespaces_content')) + namespaces = get_namespaces() + namespace_names = [ns.name for ns in namespaces] + logger.info(f'Namespaces found:\n {namespace_list_to_str(namespaces)}') + logger.info(f'Source namespaces found:\n {namespace_list_to_str(source_namespaces)}') + + bgd = get_bgd_object() + logger.info(f'BGD object: {bgd}') + + filtered_namespaces = filter_namespaces(namespace_names, filter, bgd) + namespaces_to_restore = [ns for ns in namespaces if ns.name not in filtered_namespaces] + logger.info(f"Namespaces that didn't pass filter will be restored:\n {namespace_list_to_str(namespaces_to_restore)}") + + namespace_paths_to_restore = [ns for ns in namespaces_to_restore] + + if namespace_paths_to_restore: + for ns in namespace_paths_to_restore: + restore_namespace(ns, source_namespaces) + logger.info(f"Restoration was successful") + else: + logger.info(f"No namespaces to restore") + +def restore_namespace(namespace: Namespace, source_namespaces: list[Namespace]): + shutil.rmtree(namespace.path) + source_namespace = next((sns for sns in source_namespaces if sns.name == namespace.name), None) + if not source_namespace: + logger.debug(f'Source namespace for {namespace.name} not found, just removed namespace') + return + logger.debug(f'Restored {namespace.name}') + shutil.copytree(source_namespace.path, namespace.path, dirs_exist_ok=True) + +def namespace_list_to_str(namespaces: list[Namespace]): + return "\n".join(f" - {ns.name} {ns.path}" for ns in namespaces) + + + diff --git a/scripts/build_env/main.py b/scripts/build_env/main.py index 08caa9f6..3a09c82a 100644 --- a/scripts/build_env/main.py +++ b/scripts/build_env/main.py @@ -10,6 +10,8 @@ from resource_profiles import get_env_specific_resource_profiles from pathlib import Path +from filter_namespaces import apply_ns_build_filter + # const INVENTORY_DIR_NAME = "Inventory" ENV_DEFINITION_FILE_NAME = "env_definition.yml" @@ -27,7 +29,7 @@ def prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templa delete_dir(render_profiles_dir) render_env_dir = f"{render_dir}/{env_name}" copy_path(f'{source_env_dir}/{INVENTORY_DIR_NAME}', f"{render_env_dir}/{INVENTORY_DIR_NAME}") - # clearing instances dir + # clearing instances dir cleanup_resulting_dir(Path(output_dir) / cluster_name / env_name) # copying parameters from templates and instances check_dir_exist_and_create(f'{render_parameters_dir}/from_template') @@ -104,6 +106,11 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all render_parameters_dir = getAbsPath('tmp/parameters_templates') render_profiles_dir = getAbsPath('tmp/resource_profiles') + namespaces_path = get_namespaces_path() + if check_dir_exists(str(namespaces_path.absolute())): + logger.info("Namespaces found, saving them into tmp location") + shutil.copytree(get_namespaces_path(), os.path.join(work_dir,'build_env','tmp','initial_namespaces_content','Namespaces'), dirs_exist_ok=True) + # preparing folders for generation render_env_dir = prepare_folders_for_rendering(env_name, cluster_name, source_env_dir, templates_dir, render_dir, render_parameters_dir, render_profiles_dir, output_dir) @@ -404,6 +411,7 @@ def render_environment(env_name, cluster_name, templates_dir, all_instances_dir, create_credentials(resulting_env_dir, env_dir, all_instances_dir) # update versions update_generated_versions(resulting_env_dir, BUILD_ENV_TAG, g_template_version) + apply_ns_build_filter() if __name__ == "__main__": diff --git a/scripts/build_env/test_namespace_filter.py b/scripts/build_env/test_namespace_filter.py new file mode 100644 index 00000000..beec70c7 --- /dev/null +++ b/scripts/build_env/test_namespace_filter.py @@ -0,0 +1,43 @@ +import pytest +from filter_namespaces import filter_namespaces + +@pytest.fixture +def sample_namespaces(): + return [ 'ns1','ns2','ns3'] + +@pytest.fixture +def sample_bgd(): + return { + "peerNamespace": {"name": "ns1"}, + "originNamespace": {"name": "ns3"}, + } + +def test_no_filter_returns_all(sample_namespaces, sample_bgd): + result = filter_namespaces(sample_namespaces, "", sample_bgd) + assert len(result) == 3 + assert {ns for ns in result} == {"ns1", "ns2", "ns3"} + +def test_inclusion_filter_by_name(sample_namespaces, sample_bgd): + result = filter_namespaces(sample_namespaces, "ns1,ns3", sample_bgd) + assert len(result) == 2 + assert {ns for ns in result} == {"ns1", "ns3"} + +def test_exclusion_filter_by_name(sample_namespaces, sample_bgd): + result = filter_namespaces(sample_namespaces, "!ns2", sample_bgd) + assert len(result) == 2 + assert {ns for ns in result} == {"ns1", "ns3"} + +def test_inclusion_filter_with_alias(sample_namespaces, sample_bgd): + result = filter_namespaces(sample_namespaces, "@peer,@origin", sample_bgd) + assert len(result) == 2 + assert {ns for ns in result} == {"ns1", "ns3"} + +def test_exclusion_filter_with_alias(sample_namespaces, sample_bgd): + result = filter_namespaces(sample_namespaces, "!@peer", sample_bgd) + assert len(result) == 2 + assert {ns for ns in result} == {"ns2", "ns3"} + +def test_invalid_alias_raises_error(sample_namespaces, sample_bgd): + with pytest.raises(ValueError) as exc: + filter_namespaces(sample_namespaces, "@unknown", sample_bgd) + assert "Unknown alias" in str(exc.value) diff --git a/scripts/build_env/test_render_envs.py b/scripts/build_env/test_render_envs.py index 312b1e44..b6b93cee 100644 --- a/scripts/build_env/test_render_envs.py +++ b/scripts/build_env/test_render_envs.py @@ -17,7 +17,8 @@ ("cluster01", "env01", "test-01"), ("cluster01", "env01", "test-01"), ("cluster01", "env03", "test-template-1"), - ("cluster01", "env04", "test-template-2") + ("cluster01", "env04", "test-template-2"), + ("bgd-cluster","bgd-env","bgd"), ] g_templates_dir = getAbsPath("../../test_data/test_templates") @@ -35,6 +36,7 @@ def change_test_dir(request, monkeypatch): @pytest.mark.parametrize("cluster_name, env_name, version", test_data) def test_render_envs(cluster_name, env_name, version): environ['CI_PROJECT_DIR'] = g_base_dir + environ['FULL_ENV_NAME'] = cluster_name + '/' + env_name render_environment(env_name, cluster_name, g_templates_dir, g_inventory_dir, g_output_dir, version, g_base_dir) source_dir = f"{g_inventory_dir}/{cluster_name}/{env_name}" generated_dir = f"{g_output_dir}/{cluster_name}/{env_name}" @@ -71,7 +73,7 @@ def test_render_envs(cluster_name, env_name, version): def setup_test_dir(tmp_path): - tmp_path.mkdir(exist_ok=True) + tmp_path.mkdir(exist_ok=True, parents=True) dirs = ["Applications", "Namespaces", "Profiles"] for d in dirs: (tmp_path / d).mkdir(exist_ok=True) diff --git a/scripts/build_env/validate_bgd.py b/scripts/build_env/validate_bgd.py new file mode 100644 index 00000000..6586d067 --- /dev/null +++ b/scripts/build_env/validate_bgd.py @@ -0,0 +1,21 @@ +from envgenehelper.business_helper import get_bgd_object, get_namespaces +from envgenehelper import logger + +def main(): + logger.info(f'Validating that all namespaces mentioned in BG domain object are available in namespaces') + namespace_names = [ns.name for ns in get_namespaces()] + bgd = get_bgd_object() + mismatch = "" + for k,v in bgd.items(): + if not 'Namespace' in k: + continue + if v['name'] not in namespace_names: + mismatch += (f"\n{v['name']} from {k}") + if mismatch: + logger.info(f'Available namespaces: {namespace_names}') + raise ValueError(f'Next namespaces were not found in available namespaces: {mismatch}') + logger.info(f'Validation was successful') + + +if __name__ == "__main__": + main() diff --git a/test_data/test_environments/bgd-cluster/bgd-env/Inventory/env_definition.yml b/test_data/test_environments/bgd-cluster/bgd-env/Inventory/env_definition.yml new file mode 100644 index 00000000..e238dea8 --- /dev/null +++ b/test_data/test_environments/bgd-cluster/bgd-env/Inventory/env_definition.yml @@ -0,0 +1,17 @@ +inventory: + environmentName: "bgd-env" + clusterUrl: "test-val.com" + tenantName: "test-tenant" + deployer: "test-deployer" + cloudName: "test-solution-structure" + cloudPassport: "test-cloud-passport" +envTemplate: + name: "bgd" + additionalTemplateVariables: {} + sharedTemplateVariables: [] + envSpecificParamsets: {} + envSpecificTechnicalParamsets: {} + sharedMasterCredentialFiles: [] + artifact: "bgd" +generatedVersions: + generateEnvironmentLatestVersion: "bgd:bgd" # This value is automatically generated during job run. diff --git a/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml b/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml index 364ae889..abaf90cd 100644 --- a/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml +++ b/test_data/test_environments/cluster-01/cloud-passport/cluster-01-creds.yml @@ -2,22 +2,22 @@ cloud-deploy-sa-token: type: "secret" data: secret: "eyJhbGciOiJSUzI1NiIsImtpZCI6Ilh6bE9fS19OaC1LZ2ZJNk5fVXlIV18yWFh1V19oYlJqV2FfR2JtY3MifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6ImRlZmF1bHQtdG9rZW4teHh4eHgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGVmYXVsdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjEyMzQ1Ni0xMjM0LTEyMzQtMTIzNC0xMjM0NTY3ODkiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6ZGVmYXVsdDpkZWZhdWx0In0.AB12CD34EF56GH78IJ90KL12MN34OP56QR78ST90UV12WX34YZ56AB78CD90EF12GH34IJ56KL78MN90OP12QR34ST56UV78WX90YZ12AB34CD56EF78GH90IJ12KL34MN56OP78QR90ST12UV34WX56YZ78AB90CD12EF34GH56IJ78KL90MN12OP34QR56ST78UV90WX12YZ34AB56CD78EF90GH12IJ34KL56MN78OP90QR12ST34UV56WX78YZ90" +consul-cred: + type: "secret" + data: + secret: "b2c4d6e8-f1a3-5b7d-9e0f-1a2b3c4d5e6f" dbaas-cred: - type: usernamePassword + type: "usernamePassword" data: - username: dbaas-username - password: dbaas-password + username: "dbaas-username" + password: "dbaas-password" maas-cred: - type: usernamePassword - data: - username: maas-username - password: maas-password -consul-cred: - type: secret + type: "usernamePassword" data: - secret: b2c4d6e8-f1a3-5b7d-9e0f-1a2b3c4d5e6f + username: "maas-username" + password: "maas-password" minio-cred: - type: usernamePassword + type: "usernamePassword" data: - username: minio-username - password: minio-password + username: "minio-username" + password: "minio-password" diff --git a/test_data/test_environments/cluster01/env01/Credentials/credentials.yml b/test_data/test_environments/cluster01/env01/Credentials/credentials.yml index b0877d44..fcf1842a 100644 --- a/test_data/test_environments/cluster01/env01/Credentials/credentials.yml +++ b/test_data/test_environments/cluster01/env01/Credentials/credentials.yml @@ -1,3 +1,7 @@ +bgd-controller-token: + type: "secret" + data: + secret: "envgeneNullValue" # FillMe cloud-deploy-sa-token: # cloud passport: test-cloud-passport version: 1.5 type: "secret" data: diff --git a/test_data/test_templates/env_templates/bgd.yaml b/test_data/test_templates/env_templates/bgd.yaml new file mode 100644 index 00000000..a1af6096 --- /dev/null +++ b/test_data/test_templates/env_templates/bgd.yaml @@ -0,0 +1,21 @@ +--- +tenant: "{{ templates_dir }}/env_templates/bgd/tenant.yml.j2" +cloud: + template_path: "{{ templates_dir }}/env_templates/bgd/cloud.yml.j2" +namespaces: + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/app.yml.j2" + deploy_postfix: "peer-app" + template_override: + name: "{{current_env.name}}-peer-app" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/app.yml.j2" + deploy_postfix: "origin-app" + template_override: + name: "{{current_env.name}}-origin-app" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-plugin.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-controller.yml.j2" + - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/other-app.yml.j2" +bg_domain: "{{ templates_dir }}/env_templates/bgd/bg_domain.yml.j2" +parametersets: [] + + + diff --git a/test_data/test_templates/env_templates/bgd/Namespaces/app.yml.j2 b/test_data/test_templates/env_templates/bgd/Namespaces/app.yml.j2 new file mode 100644 index 00000000..3530dedb --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/Namespaces/app.yml.j2 @@ -0,0 +1,19 @@ +--- +name: "{{current_env.name}}-app" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" +e2eParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +technicalConfigurationParameters: + APP_NAMESPACE: "{{current_env.name}}-app" +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/bgd/Namespaces/bg-controller.yml.j2 b/test_data/test_templates/env_templates/bgd/Namespaces/bg-controller.yml.j2 new file mode 100644 index 00000000..6ad1bcad --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/Namespaces/bg-controller.yml.j2 @@ -0,0 +1,14 @@ +--- +name: "{{current_env.name}}-bg-controller" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: {} +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/bgd/Namespaces/bg-plugin.yml.j2 b/test_data/test_templates/env_templates/bgd/Namespaces/bg-plugin.yml.j2 new file mode 100644 index 00000000..74286d02 --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/Namespaces/bg-plugin.yml.j2 @@ -0,0 +1,14 @@ +--- +name: "{{current_env.name}}-bg-plugin" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: {} +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/bgd/Namespaces/other-app.yml.j2 b/test_data/test_templates/env_templates/bgd/Namespaces/other-app.yml.j2 new file mode 100644 index 00000000..cad3962d --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/Namespaces/other-app.yml.j2 @@ -0,0 +1,17 @@ +--- +name: "{{current_env.name}}-other-app" +labels: +- "Instance-{{current_env.name}}" +credentialsId: "" +isServerSideMerge: false +cleanInstallApprovalRequired: false +mergeDeployParametersAndE2EParameters: false +deployParameters: + ENVGENE_CONFIG_REF_NAME: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_REF_NAME')| default('No Ref Name') }}" + ENVGENE_CONFIG_TAG: "{{ lookup('ansible.builtin.env', 'CI_COMMIT_TAG')| default('No Ref tag') }}" + APP_NAMESPACE: "{{current_env.name}}-app" +e2eParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 b/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 new file mode 100644 index 00000000..a239f773 --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 @@ -0,0 +1,13 @@ +name: "{{current_env.name}}-bg-domain" +type: bgdomain +originNamespace: + name: "{{current_env.name}}-origin-bss" + type: namespace +peerNamespace: + name: "{{current_env.name}}-peer-bss" + type: namespace +controllerNamespace: + name: "{{current_env.name}}-bg-controller" + type: namespace + credentials: bgdomaincred + diff --git a/test_data/test_templates/env_templates/bgd/cloud.yml.j2 b/test_data/test_templates/env_templates/bgd/cloud.yml.j2 new file mode 100644 index 00000000..38539a8f --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/cloud.yml.j2 @@ -0,0 +1,39 @@ +--- +name: "{{current_env.cloud}}" +apiUrl: "{{current_env.cluster.cloud_api_url}}" +apiPort: "{{current_env.cluster.cloud_api_port}}" +privateUrl: "" +publicUrl: "{{current_env.cluster.cloud_public_url}}" +dashboardUrl: "https://dashboard.{{current_env.cluster.cloud_public_url}}" +labels: [] +protocol: "{{current_env.cluster.cloud_api_protocol}}" +dbMode: "db" +databases: [] +defaultCredentialsId: "admin-token" +maasConfig: + credentialsId: "maas" + maasUrl: "http://maas.{{current_env.cluster.cloud_public_url}}" + maasInternalAddress: "http://maas.maas:8888" + enable: true +vaultConfig: + url: "" + credentialsId: "" + enable: false +dbaasConfigs: + - credentialsId: "dbaas" + apiUrl: 'http://dbaas.dbaas:8888' + aggregatorUrl: 'https://dbaas.{{current_env.cluster.cloud_public_url}}' + enable: true +consulConfig: + tokenSecret: "consul-token" + publicUrl: 'https://consul.{{current_env.cluster.cloud_public_url}}' + enabled: true + internalUrl: 'http://consul.consul:8888' +e2eParameters: + CMDB_NAME: "{{current_env.cmdb_name}}" + TEMPLATE_NAME: "{{current_env.env_template}}" +deployParameters: {} +technicalConfigurationParameters: {} +deployParameterSets: [] +e2eParameterSets: [] +technicalConfigurationParameterSets: [] diff --git a/test_data/test_templates/env_templates/bgd/tenant.yml.j2 b/test_data/test_templates/env_templates/bgd/tenant.yml.j2 new file mode 100644 index 00000000..11cfdf42 --- /dev/null +++ b/test_data/test_templates/env_templates/bgd/tenant.yml.j2 @@ -0,0 +1,14 @@ +name: "{{current_env.tenant}}" +registryName: "default" +description: "{{current_env.description}}" +owners: "{{current_env.owners}}" +deployParameters: {} +globalE2EParameters: + pipelineDefaultRecipients: "" + recipientsStrategy: "merge" + mergeTenantsAndE2EParameters: false + environmentParameters: {} +gitRepository: "" +defaultBranch: "" +credential: "" +labels: [] diff --git a/test_data/test_templates/env_templates/test-solution-structure-template.yaml b/test_data/test_templates/env_templates/test-solution-structure-template.yaml index 33f368ca..b9ef0f12 100644 --- a/test_data/test_templates/env_templates/test-solution-structure-template.yaml +++ b/test_data/test_templates/env_templates/test-solution-structure-template.yaml @@ -21,4 +21,5 @@ namespaces: - paramset-A - template_path: "{{ templates_dir }}/env_templates/test-solution-structure-template/Namespaces/app.yml.j2" deploy_postfix: "app" +bg_domain: "{{ templates_dir }}/env_templates/test-solution-structure-template/bg_domain.yml.j2" diff --git a/test_data/test_templates/env_templates/test-solution-structure-template/bg_domain.yml.j2 b/test_data/test_templates/env_templates/test-solution-structure-template/bg_domain.yml.j2 new file mode 100644 index 00000000..4ce549c2 --- /dev/null +++ b/test_data/test_templates/env_templates/test-solution-structure-template/bg_domain.yml.j2 @@ -0,0 +1,12 @@ +name: {{ current_env.environmentName ~ '-bg-domain' }} +type: bgdomain +originNamespace: + name: app-core + type: namespace +peerNamespace: + name: app + type: namespace +controllerNamespace: + name: app + type: namespace + credentials: bgd-controller-token