feat: NS filter support

.githooks/commit-msg

@@ -0,0 +1,12 @@
+#!/bin/sh
+msg_file="$1"
+first_line=$(head -n 1 "$msg_file")
+
+# Allowed prefixes (extend if needed)
+case "$first_line" in
+  fix:*|feat:*|chore:*|docs:*|refactor:*|test:*|ci:*)
+    exit 0 ;; # ok
+esac
+
+# Prepend "fix:" if no prefix
+sed -i "1s|^|fix: |" "$msg_file"

.gitignore

@@ -8,3 +8,5 @@ venv
 __pycache__
 .vscode
 .DS_Store
+pyrightconfig.json
+/qwerty

Makefile

@@ -0,0 +1,27 @@
+compose := sudo docker compose -f devtools/docker-compose.yml
+
+build-%:
+	$(compose) build $*
+
+up-%:
+	$(compose) up -d $*
+	@if [ -f devtools/$*/up.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/up.sh; fi
+
+bash-%:
+	$(compose) exec $* bash
+
+down:
+	$(compose) down
+
+stop-%:
+	$(compose) stop $*
+
+rm-%:
+	$(compose) rm $*
+
+run-%:
+	@if [ -f devtools/$*/run.sh ]; then $(compose) exec $* sh /workspace/devtools/$*/run.sh; \
+	else echo "No run script for $*"; fi
+
+edit:
+	vim $(abspath $(lastword $(MAKEFILE_LIST)))

build_pipegene/scripts/env_build_jobs.py

@@ -21,6 +21,8 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full
   else:
      script.append("export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')")
 
+  script.append("python /build_env/scripts/build_env/validate_bgd.py")
+  script.append("python /build_env/scripts/build_env/filter_namespaces.py")
   script.extend([
       'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")',
       'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi;  done'
@@ -41,6 +43,7 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full
 
   env_build_vars = {
       "ENV_NAME": full_env,
+      "FULL_ENV_NAME": full_env,
       "CLUSTER_NAME": cluster_name,
       "ENVIRONMENT_NAME": enviroment_name,
       "ENV_TEMPLATE_VERSION": env_template_version,
@@ -138,6 +141,7 @@ def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, de
   git_commit_job = job_instance(params=git_commit_params, vars=git_commit_vars)
   git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/" + f"{full_env}")
   git_commit_job.artifacts.add_paths("${CI_PROJECT_DIR}/git_envs")
+  git_commit_job.artifacts.add_paths('${CI_PROJECT_DIR}/sboms')
   git_commit_job.artifacts.when = WhenStatement.ALWAYS
   if (credential_rotation_job is not None):
     git_commit_job.add_needs(credential_rotation_job)

build_pipegene/scripts/gitlab_ci.py

@@ -22,7 +22,7 @@
 def build_pipeline(params: dict):
     # if we are in template testing during template build
     tags=params['GITLAB_RUNNER_TAG_NAME']
-    
+
     if params['IS_TEMPLATE_TEST']:
         logger.info("We are generating jobs in template test mode.")
         templates_dir = f"{project_dir}/templates/env_templates"
@@ -117,21 +117,23 @@ def build_pipeline(params: dict):
         else:
             logger.info(f'Preparing of generate_effective_set job for {cluster_name}/{environment_name} is skipped.')
 
-        ## git_commit job
-        jobs_requiring_git_commit = ("env_build_job", "generate_effective_set_job", "env_inventory_generation_job", "credential_rotation_job")
-        if any(job in jobs_map for job in jobs_requiring_git_commit) and not params['IS_TEMPLATE_TEST']:
-            jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, env, environment_name, cluster_name, params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job)
-        else:
-            logger.info(f'Preparing of git commit job for {env} is skipped.')
+        jobs_requiring_git_commit = ["env_build_job", "generate_effective_set_job", "env_inventory_generation_job", "credential_rotation_job"]
 
         plugin_params = params
         plugin_params['jobs_map'] = jobs_map
         plugin_params['job_sequence'] = job_sequence
+        plugin_params['jobs_requiring_git_commit'] = jobs_requiring_git_commit
         plugin_params['env_name'] = environment_name
         plugin_params['cluster_name'] = cluster_name
         plugin_params['full_env'] = env
         per_env_plugin_engine.run(params=plugin_params, pipeline=pipeline, pipeline_helper=pipeline_helper)
 
+        ## git_commit job
+        if any(job in jobs_map for job in plugin_params['jobs_requiring_git_commit']) and not params['IS_TEMPLATE_TEST']:
+            jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, env, environment_name, cluster_name, params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job)
+        else:
+            logger.info(f'Preparing of git commit job for {env} is skipped.')
+
         for job in job_sequence:
             if job not in jobs_map.keys():
                 continue

build_pipegene/scripts/pipeline_parameters.py

@@ -28,8 +28,9 @@ def get_pipeline_parameters() -> dict:
         },
         'CRED_ROTATION_PAYLOAD': getenv("CRED_ROTATION_PAYLOAD", ""),
         'CRED_ROTATION_FORCE': getenv("CRED_ROTATION_FORCE", ""),
+        'NS_BUILD_FILTER': getenv("NS_BUILD_FILTER", ""),
         'GITLAB_RUNNER_TAG_NAME' : getenv("GITLAB_RUNNER_TAG_NAME", ""),
-        'RUNNER_SCRIPT_TIMEOUT' : getenv("RUNNER_SCRIPT_TIMEOUT") or "10m"
+        'RUNNER_SCRIPT_TIMEOUT' : getenv("RUNNER_SCRIPT_TIMEOUT") or "10m",
     }
 
 class PipelineParametersHandler:

devtools/docker-compose.yml

@@ -0,0 +1,19 @@
+x-base-env: &base-env
+  CI_PROJECT_DIR: /workspace
+  IS_LOCAL_DEV_TEST_ENVGENE: "true"
+
+x-base-service: &base-service
+  volumes:
+    - ../:/workspace
+  working_dir: /workspace
+  entrypoint: sleep infinity
+
+services:
+  tests:
+    <<: *base-service
+    environment:
+      <<: *base-env
+    build:
+      dockerfile: devtools/tests/Dockerfile
+      context: ../
+

devtools/readme.md

@@ -0,0 +1,43 @@
+# Guide for running things locally
+
+## Prerequisites
+
+1. You have installed `WSL` and have `docker` in it.
+
+## Setup
+
+1. Check `docker-compose.yml` file to find the service for general unit tests,
+   or job that you need to test and run
+1. Run `make build-%` and `make up-%` where `%` is name of the service to get
+   container running for service that you want. And run `make run-%` to actually
+   run the code.
+
+## Usage
+
+Format:
+
+```sh
+make <command>-<service>
+```
+
+Where `service` matches the service name from docker-compose.yml.
+
+| Command | What it does                              | Example              |
+| ------- | ----------------------------------------- | -------------------- |
+| `build` | Build service image                       | `make build-tests` |
+| `up`    | Start service (runs `up.sh` if present)   | `make up-tests`    |
+| `bash`  | Open shell in running service             | `make bash-tests`  |
+| `down`  | Stop all services                         | `make down`          |
+| `stop`  | Stop one service                          | `make stop-tests`  |
+| `rm`    | Remove service container                  | `make rm-tests`    |
+| `run`   | Run service’s `run.sh` script (if exists) | `make run-tests`   |
+| `edit`  | Edit Makefile in Vim                      | `make edit`          |
+
+**Notes:**
+
+* `up.sh` / `run.sh` are optional, per-service scripts.
+* If service is configured correctly and has the code mounted
+  into it from repos, changes made in code in repos, should reflect immediately
+  in container, so after code changes you would need to just run `make run-%`.
+  If there are changes in docker-compose - run `make up-%` again and if there
+  are changes in `Dockerfile` itself used for service - run `make build-%`

devtools/tests/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim-bookworm
+
+RUN useradd -m appuser
+USER appuser
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential=12.9 \
+    curl=7.88.1-10+deb12u8 \
+ && rm -rf /var/lib/apt/lists/*
+
+# sops
+RUN curl -LO https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 && \
+    mv sops-v3.9.0.linux.amd64 /usr/local/bin/sops && \
+    chmod +x /usr/local/bin/sops
+
+# Optional pip.conf / sources.list
+COPY dependencies/pip.conf /etc/pip.conf
+COPY dependencies/sources.list /etc/apt/sources.list
+
+# Copy dependencies and install them
+COPY dependencies/tests_requirements.txt /tmp/
+RUN pip install --no-cache-dir "uv>=0.9.5" && uv pip install --system --no-cache-dir -r /tmp/tests_requirements.txt
+

devtools/tests/run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euxo
+
+cd "$CI_PROJECT_DIR"
+
+# Run tests
+cd python/envgene/envgenehelper
+pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../../junit.xml
+cd ../../..
+mv junit.xml junit_envgenehelper.xml
+
+cd scripts/build_env
+pytest --capture=no -W ignore::DeprecationWarning --junitxml=../../junit.xml
+cd ../..
+mv junit.xml junit_build_env.xml
+
+# Merge results
+python -m junitparser merge junit_build_env.xml junit_envgenehelper.xml junit.xml

devtools/tests/up.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+chmod +x /workspace/python/build_modules.sh
+/workspace/python/build_modules.sh

docs/instance-pipeline-parameters.md

@@ -26,6 +26,9 @@
     - [`CRED_ROTATION_PAYLOAD`](#cred_rotation_payload)
       - [Affected Parameters and Troubleshooting](#affected-parameters-and-troubleshooting)
     - [`CRED_ROTATION_FORCE`](#cred_rotation_force)
+    - [`SD_REPO_MERGE_MODE`](#sd_repo_merge_mode)
+    - [`NS_BUILD_FILTER`](#ns_build_filter)
+    - [`GITHUB_PIPELINE_API_INPUT`](#github_pipeline_api_input)
     - [`GH_ADDITIONAL_PARAMS`](#gh_additional_params)
   - [Deprecated Parameters](#deprecated-parameters)
     - [`SD_DELTA`](#sd_delta)

env-builder/main.yaml

@@ -118,6 +118,17 @@
       loop_control:
         loop_var: namespace
 
+    - name: Gen BG Domain
+      ansible.builtin.include_role:
+        name: generate_bgd
+      vars:
+        _env: "{{ env }}"
+        _tenant: "{{ tenant }}"
+        _cloud: "{{ current_env.cloud }}"
+        _current_env_dir: "{{ current_env_dir }}"
+        _template: "{{current_env_template.bg_domain}}"
+      when: current_env_template.bg_domain is defined
+
     - name: Gen resource profiles
       ansible.builtin.include_role:
         name: generate_profiles
@@ -263,4 +274,3 @@
         not use_external_defs and
         ((appdef_templates.files is defined and appdef_templates.files | length > 0) or
         (regdef_templates.files is defined and regdef_templates.files | length > 0))
-

env-builder/roles/generate_bgd/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Generate BG Domain yaml {{ _bg_domain }}
+  ansible.builtin.blockinfile:
+     path: "{{ current_env_dir }}/bg_domain.yml"
+     block: "{{ lookup('template', _template) }}"
+     create: yes

github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml

@@ -69,9 +69,9 @@ env:
   DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator"
 
   #DOCKER_IMAGE_TAGS
-  DOCKER_IMAGE_TAG_PIPEGENE: "1.6.5"
-  DOCKER_IMAGE_TAG_ENVGENE: "1.6.5"
-  DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.6.5"
+  DOCKER_IMAGE_TAG_PIPEGENE: "feature_ns-filter-support_20251028-053643"
+  DOCKER_IMAGE_TAG_ENVGENE: "feature_ns-filter-support_20251028-053639"
+  DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "feature_ns-filter-support_20251028-053736"
 
 jobs:
   process_environment_variables:

python/build_modules.sh

@@ -9,10 +9,15 @@ install_and_clean() {
     local base_path="${3:-$SCRIPTPATH}"  # optional third argument to function
 
     echo "Installing $base_path/$path"
-    if [ "$IS_LOCAL_DEV_TEST_ENVGENE" = "true" ]; then
-      uv pip install --editable "$base_path/$path"
+    if [ "$USE_UV" = "true" ]; then
+        echo "USE_UV is true, using uv"
+        if ! command -v uv &>/dev/null; then
+            echo "uv not found, installing..."
+            pip install uv
+        fi
+        uv pip install --editable "$base_path/$path"
     else
-      pip install "$base_path/$path"
+        pip install "$base_path/$path"
     fi
 
     echo "Removing build trash..."