feat: Added CRED ROTATION

.gitignore

@@ -7,3 +7,4 @@ venv
 /env-builder/deployers
 __pycache__
 .vscode
+.DS_Store

build_effective_set_generator/scripts/test_generate_effective_set.py

@@ -5,7 +5,10 @@
 from envgenehelper import *
 
 test_data = [
-      ("CLUSTER", "ENV", ""),
+      ("REPLACED_WORD-ocp-mdc-09", "cse-toolset", "")
+    , ("REPLACED_WORD-sb-ocp-01", "pl01", "")
+    , ("REPLACED_WORD-sb-ocp-01", "platform-with-overrides", "")
+    , ("cloud-with-passport-override", "cse-toolset", "")
 ]
 
 g_inventory_dir = getAbsPath("../../test_data/test_environments")

build_envgene/ansible/roles/build_env/tasks/02_prepare_vars.yaml

@@ -25,4 +25,4 @@
 
 - name: include new logic
   include_tasks: 02_prepare_vars_new.yaml
-  when: env_definition['envTemplate']['artifact'] is defined
+  when: env_definition['envTemplate']['artifact'] is defined

build_envgene/ansible/roles/build_env/tasks/02_prepare_vars_new.yaml

@@ -39,6 +39,8 @@
     extension: json
   no_log: false
   register: discovery_output
+  when:
+    - not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 - name: Set app template
   set_fact:

build_envgene/ansible/roles/build_env/tasks/03_upload_template.yaml

@@ -9,7 +9,7 @@
     password:       "{{ repository_password }}"
     dest:           "{{ dd_dest }}"
     extension: json
-  when: not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+  when: not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 - name: Run maven artifact to find artifact version
   maven_artifact:
@@ -24,39 +24,39 @@
   no_log: false
   when:
     - version is search('SNAPSHOT')
-    - not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+    - not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
   register: module_output
 
 - debug:
     var: module_output.version_dd
   when:
     - version is search('SNAPSHOT')
-    - not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+    - not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 - name: Set artifact version from snapshot
   set_fact:
     atrifact_latest_version: "{{ module_output.version_dd  | default(version) }}"
-  when: not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+  when: not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 - name: Lookup artifact version from downloaded artifact
   set_fact:
     atrifact_name_list: "{{ lookup('file',lookup('ansible.builtin.vars','dd_dest')) | from_json | json_query('configurations[0].artifacts[0].id')| split(':') }}"
-  when: not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+  when: not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 - name: Lookup template repository from downloaded artifact
   set_fact:
     template_repository_url: "{{ (lookup('file',lookup('ansible.builtin.vars','dd_dest')) | from_json | json_query('configurations[0].maven_repository')) or repository_url }}"
   when:
     - env_definition['envTemplate']['artifact'] is defined
-    - not (env_definition.envTemplate.template_download_zip | default(false) | bool)
+    - not (env_definition.envTemplate.artifactIsZip | default(false) | bool)
 
 
 - name: Download a real artifact using Artifactory username/password
   community.general.maven_artifact:
     group_id:       "{{ atrifact_name_list[0] | default(group_id) }}"
     artifact_id:    "{{ atrifact_name_list[1] | default(artifact_id) }}"
     version:        "{{ atrifact_name_list[2] | default(version)  }}"
-    repository_url: "{{ template_repository_url }}"
+    repository_url: "{{ template_repository_url | default(repository_url) }}"
     username:       "{{ repository_username }}"
     password:       "{{ repository_password }}"
     dest:           "{{ artifact_dest }}"
@@ -66,4 +66,3 @@
   unarchive:
     src: "{{ artifact_dest }}"
     dest: "{{ build_env_path }}"
-

build_envgene/build/Dockerfile

@@ -1,59 +1,161 @@
 #########################################
-## Stage 1
-FROM ghcr.io/netcracker/base-images-module-base:main as base-image
+# Stage 1: Build
+# Multi-stage build to reduce final image size
+FROM python:3.12-alpine3.19 AS build
 
-USER root
+# Install build dependencies
+RUN apk add --no-cache \
+    gcc \
+    musl-dev \
+    libffi-dev \
+    openssl-dev \
+    libxml2-dev \
+    libxslt-dev \
+    zlib-dev \
+    git \
+    curl \
+    jq \
+    openssh-client \
+    sudo \
+    zip \
+    unzip
 
+# Copy configuration files
 COPY build_envgene/build/pip.conf /etc/pip.conf
 COPY build_envgene/build/requirements.txt /build/requirements.txt
 COPY build_envgene/build/requirements.yml /build/requirements.yml
+COPY build_envgene/build/constraint.txt /build/constraint.txt
+COPY creds_rotation/build/requirements.txt /build/creds_rotation_requirements.txt
+
+# Copy source code
+COPY python /python
 COPY build_envgene/ansible /module/ansible
 COPY build_envgene/scripts /module/scripts
-COPY build_envgene/workflows /workflows
-COPY python /python
-
+COPY creds_rotation/scripts /module/creds_rotation_scripts
 COPY build_* create_* produce_* sort* /build_env/
 COPY scripts /build_env/scripts
 COPY env-builder /build_env/env-builder
 COPY schemas /build_env/schemas
 
+ENV ANSIBLE_LIBRARY=/module/ansible/library
+
+# Create virtual environment and install Python packages
+RUN python -m venv /module/venv
+RUN /module/venv/bin/pip install --upgrade pip setuptools wheel
+RUN /module/venv/bin/pip install --no-cache-dir --retries 10 --timeout 60 -r /build/requirements.txt
+RUN /module/venv/bin/pip install ansible-core --upgrade
+# Install essential Ansible collections
+# Install to virtual environment site-packages for Python module access
+RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/venv/lib/python3.12/site-packages/ansible_collections
+RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/venv/lib/python3.12/site-packages/ansible_collections
+RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/venv/lib/python3.12/site-packages/ansible_collections
+# Also install to custom location for playbook usage
+RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/ansible/collections
+RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/ansible/collections
+RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/ansible/collections
+
+RUN /module/venv/bin/pip install /python/jschon-sort
+RUN /module/venv/bin/pip install /python/envgene
+RUN /module/venv/bin/pip install /python/integration
+RUN /module/venv/bin/pip install --no-cache-dir --no-deps -r /build/creds_rotation_requirements.txt
+
+# Download and install SOPS for secrets management
+RUN wget --tries=3 \
+    https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \
+    -O /usr/local/bin/sops && \
+    chmod +x /usr/local/bin/sops
+
+# Aggressive cleanup to reduce image size
+RUN apk del gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev zlib-dev
+RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache
+# Remove unnecessary files from Python packages
+RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete
+# Don't remove test directories as they might be needed by Ansible
+RUN find /module/venv/lib/python3.12/site-packages -name '*.pyo' -delete
+RUN find /module/venv/lib/python3.12/site-packages -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true
+# Remove heavy Ansible collections that are not essential (but keep ansible.posix and ansible.utils)
+RUN rm -rf /module/venv/lib/python3.12/site-packages/ansible_collections/amazon /module/venv/lib/python3.12/site-packages/ansible_collections/azure /module/venv/lib/python3.12/site-packages/ansible_collections/google /module/venv/lib/python3.12/site-packages/ansible_collections/kubernetes 2>/dev/null || true
+# Remove test packages that are not needed in runtime (but keep Ansible test files)
+RUN rm -rf /module/venv/lib/python3.12/site-packages/pytest* /module/venv/lib/python3.12/site-packages/_pytest* 2>/dev/null || true
+RUN /module/venv/bin/pip cache purge
+
+# Verify collections are still accessible after cleanup
+RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection still accessible after cleanup')"
+
+# Set permissions
+RUN chmod 754 /module/scripts/*
+RUN chmod 754 /module/creds_rotation_scripts/*
+
+#########################################
+# Stage 2: Runtime
+# Lightweight runtime image with only essential dependencies
+FROM python:3.12-alpine3.19 AS runtime
+
+# Install only essential runtime dependencies
+RUN apk add --no-cache \
+    bash \
+    ca-certificates \
+    curl \
+    jq \
+    yq \
+    gettext \
+    age \
+    git \
+    openssh-client \
+    sudo \
+    zip \
+    unzip
+
+# Copy everything from build stage
+COPY --from=build /module /module
+COPY --from=build /usr/local/bin/sops /usr/local/bin/sops
+COPY --from=build /build_env /build_env
+COPY --from=build /python /python
+COPY --from=build /etc/pip.conf /etc/pip.conf
+
+# Verify collections are accessible in runtime stage
+RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection accessible in runtime')"
+
+
 
-ENV ANSIBLE_LIBRARY /module/ansible/library:$ANSIBLE_LIBRARY
-
-
-RUN apk update \
-    && apk add --no-cache \
-        git \
-        python3 \
-        py3-pip \
-        py3-cryptography \
-        py3-openssl \
-        openssh-client \
-        ca-certificates \
-        coreutils \
-        curl \
-        sudo \
-        zip \
-        unzip \
-        build-base \
-        libxml2-dev \
-        libxslt-dev \
-        zlib-dev \
-        jq \
-    && curl --retry 3 --retry-connrefused --retry-delay 5 -LO https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \
-    && chmod +x sops-v3.9.0.linux.amd64 \
-    && mv sops-v3.9.0.linux.amd64 /usr/local/bin/sops \
-    && pip install --upgrade pip \
-    && pip install --no-cache-dir -r /build/requirements.txt \
-    && pip install ansible --upgrade \
-    && rm -rf /usr/lib/python3.10/site-packages/ansible_collections/* \
-    && ansible-galaxy collection install -r /build/requirements.yml -p /module/ansible/collections \
-    && pip install /python/jschon-sort \
-    && pip install /python/envgene \
-    && pip install /python/integration \
-    && chmod 754 /module/scripts/* \
-    && apk del build-base libxml2-dev libxslt-dev zlib-dev \
-    && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* \
-    && pip install PyYAML
+# Set permissions
+RUN chmod +x /usr/local/bin/sops
 
+# Create directories that might be needed for CI environments
+# These directories are commonly used by GitHub Actions and GitLab CI
+RUN mkdir -p /__w/_temp/_runner_file_commands && \
+    mkdir -p /github/workspace && \
+    mkdir -p /github/home && \
+    mkdir -p /builds && \
+    mkdir -p /cache && \
+    chmod 777 /__w/_temp/_runner_file_commands && \
+    chmod 777 /github/workspace && \
+    chmod 777 /github/home && \
+    chmod 777 /builds && \
+    chmod 777 /cache
+
+# Final cleanup
+RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache
+RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete
+# Don't remove test directories as they might be needed by Ansible
+RUN /module/venv/bin/pip cache purge
+# Keep pip for runtime compatibility, but remove setuptools and wheel
+RUN rm -rf /module/venv/lib/python3.12/site-packages/setuptools* /module/venv/lib/python3.12/site-packages/wheel* 2>/dev/null || true
+
+# Set environment
+ENV PATH=/module/venv/bin:$PATH \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    ANSIBLE_LIBRARY=/module/ansible/library \
+    ANSIBLE_COLLECTIONS_PATH=/module/venv/lib/python3.12/site-packages/ansible_collections:/module/ansible/collections
+
+# Simple root-based container for CI/CD environments
+# This container runs as root to avoid permission issues in CI/CD pipelines
 WORKDIR /module/ansible
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD python -c "import sys; sys.exit(0)" || exit 1
+
+# Default command
+CMD ["bash"]

build_envgene/build/constraint.txt

@@ -0,0 +1 @@
+cython<3 

build_envgene/build/pip.conf

@@ -1,3 +1,5 @@
 [global]
 index-url=https://pypi.org/simple
 extra-index-url=https://example.com/pypi/simple
+trusted-host=pypi.org
+# constraint=/build/constraint.txt

build_envgene/build/requirements.txt

@@ -1,16 +1,27 @@
+# Base module requirements (essential only)
+cryptography==38.0.0
+pyyaml>=6.0
+PyGithub==1.55
+certifi==2022.6.15
+
+# Build envgene requirements (essential only)
 lxml==4.9.3
 ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
-ruyaml==0.91.0
 jschon==0.11.0
 jsonschema==4.19.1
-diagrams==0.23.3
 jmespath==1.0.1
 semantic-version==2.10.0
 termcolor==2.4.0
-typing==3.7.4.3
 ansible-core==2.13.3
-ansible-base==2.10.17
-ansible_runner==2.3.5
 cffi==1.16.0
 click==8.1.3
+
+# Additional required packages
+ansible-runner==2.4.0
+
+# Removed heavy packages:
+# - shyaml, yamale, prettytable (not essential)
+# - ruyaml (duplicate of ruamel.yaml)
+# - diagrams (heavy with typed-ast dependency)
+# - ansible-base (replaced with ansible-core)

build_envgene/build/sources.list

@@ -1,2 +1,2 @@
-http://dl-cdn.alpinelinux.org/alpine/v3.16/main
-http://dl-cdn.alpinelinux.org/alpine/v3.16/community
+https://dl-cdn.alpinelinux.org/alpine/v3.20/main
+https://dl-cdn.alpinelinux.org/alpine/v3.20/community