feat(manifests): allows configuring manifest repositories using flags

Dockerfiles/Dockerfile

@@ -1,14 +1,16 @@
 # Build the manager binary
 ARG GOLANG_VERSION=1.18.9
 ARG USE_LOCAL=false
+ARG OVERWRITE_MANIFESTS=""
 
 ################################################################################
 FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_false
+ARG OVERWRITE_MANIFESTS
 # Get all manifests from remote git repo to builder_local_false by script
 USER root
 WORKDIR /opt
 COPY get_all_manifests.sh get_all_manifests.sh
-RUN ./get_all_manifests.sh
+RUN ./get_all_manifests.sh ${OVERWRITE_MANIFESTS}
 
 ################################################################################
 FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_true

README.md

@@ -43,17 +43,43 @@ Refer [Dev-Preview.md](./docs/Dev-Preview.md) for testing preview features.
 
 #### Download manifests
 
-`get_all_manifests.sh` is used to fetch manifests from remote git repos.
+The `get_all_manifests.sh` script facilitates the process of fetching manifests from remote git repositories. It is configured to work with a predefined map of components and their corresponding manifest locations.
 
-It uses a local empty folder `odh-manifests` to host all manifests operator needs, either from `odh-manifests` git repo or from component's source repo.
+#### Structure of `COMPONENT_MANIFESTS`
 
-The way to config this is to update `get_all_manifests.sh` REPO_LIST variable.
-By adding new entity in variable `REPO_LIST` in the format of `<repo-name>:<branch-name>:<source-folder>:<target-folder>` this will:
+Each component is associated with its manifest location in the `COMPONENT_MANIFESTS` map. The key is the component's name, and the value is its location, formatted as `<repo-org>:<repo-name>:<branch-name>:<source-folder>:<target-folder>`
 
-- git clone remote repo `opendatahub-io/<repo-name>` from its `<branch-name>` branch
-- copy content from its relative path `<source-folder>` into local `odh-manifests/<target-folder>` folder
+#### Workflow
 
-For those components cannot directly use manifests from `opendatahub-io/<repo-name>`, it falls back to use `opendatahub-io/odh-manifests` git repo. To control which version of `opendatahub-io/odh-manifests` to download, this is set in the `get_all_manifests.sh` variable `MANIFEST_RELEASE`.
+1. The script clones the remote repository `<repo-org>/<repo-name>` from the specified `<branch-name>`.
+2. It then copies the content from the relative path `<source-folder>` to the local `odh-manifests/<target-folder>` folder.
+
+In cases where components cannot directly use manifests from `opendatahub-io/<repo-name>`, the script defaults to the `opendatahub-io/odh-manifests` git repository.
+
+The version of manifests fetched from `opendatahub-io/odh-manifests` is determined by the `MANIFEST_RELEASE` variable in the `get_all_manifests.sh` script.
+
+#### Local Storage
+
+The script utilizes a local, empty folder named `odh-manifests` to host all required manifests, sourced either directly from the component’s source repository or the default `odh-manifests` git repository.
+
+#### Adding New Components
+
+To include a new component in the list of manifest repositories, simply extend the `COMPONENT_MANIFESTS` map with a new entry, as shown below:
+
+```shell
+declare -A COMPONENT_MANIFESTS=(
+  // existing components ...
+  ["new-component"]="<repo-org>:<repo-name>:<branch-name>:<source-folder>:<target-folder>"
+)
+```
+#### Customizing Manifests Source
+You have the flexibility to change the source of the manifests. Invoke the `get_all_manifests.sh` script with specific flags, as illustrated below:
+
+```shell
+./get_all_manifests.sh --odh-dashboard="maistra:odh-dashboard:test-manifests:manifests:odh-dashboard"
+```
+
+If the flag name matches components key defined in `COMPONENT_MANIFESTS` it will overwrite its location, otherwise the command will fail.
 
 ##### for local development
 

get_all_manifests.sh

@@ -1,47 +1,74 @@
 #!/bin/bash
 set -e
 
-# component: dsp, kserve, dashbaord, cf/ray. in the format of "repo-name:branch-name:source-folder:target-folder"
+GITHUB_URL="https://github.com/"
+# update to use different git repo for legacy manifests
+MANIFEST_ORG="opendatahub-io"
+# comment out below logic once we have all component manifests ready to get from source git repo
+MANIFEST_RELEASE="master"
+MANIFESTS_TARBALL_URL="${GITHUB_URL}/${MANIFEST_ORG}/odh-manifests/tarball/${MANIFEST_RELEASE}"
+
+# component: dsp, kserve, dashbaord, cf/ray. in the format of "repo-org:repo-name:branch-name:source-folder:target-folder"
 # TODO: workbench, modelmesh, monitoring, etc
-REPO_LIST=(
-    "distributed-workloads:main:codeflare-stack:codeflare"
-    "distributed-workloads:main:ray:ray"
-    "data-science-pipelines-operator:main:config:data-science-pipelines-operator"
-    "odh-dashboard:main:manifests:odh-dashboard"
-    "kubeflow:v1.7-branch:components/notebook-controller/config:odh-notebook-controller/kf-notebook-controller"
-    "kubeflow:v1.7-branch:components/odh-notebook-controller/config:odh-notebook-controller/odh-notebook-controller"
-    "notebooks:main:manifests:notebooks"
+declare -A COMPONENT_MANIFESTS=(
+    ["codeflare"]="opendatahub-io:distributed-workloads:main:codeflare-stack:codeflare"
+    ["ray"]="opendatahub-io:distributed-workloads:main:ray:ray"
+    ["data-science-pipelines-operator"]="opendatahub-io:data-science-pipelines-operator:main:config:data-science-pipelines-operator"
+    ["odh-dashboard"]="opendatahub-io:odh-dashboard:main:manifests:odh-dashboard"
+    ["kf-notebook-controller"]="opendatahub-io:kubeflow:v1.7-branch:components/notebook-controller/config:odh-notebook-controller/kf-notebook-controller"
+    ["odh-notebook-controller"]="opendatahub-io:kubeflow:v1.7-branch:components/odh-notebook-controller/config:odh-notebook-controller/odh-notebook-controller"
+    ["notebooks"]="opendatahub-io:notebooks:main:manifests:notebooks"
 )
 
+# Allow overwriting repo using flags component=repo
+pattern="^[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_./-]+:[a-zA-Z0-9_./-]+$"
+if [ "$#" -ge 1 ]; then
+    for arg in "$@"; do
+        if [[ $arg == --* ]]; then
+            arg="${arg:2}"  # Remove the '--' prefix
+            IFS="=" read -r key value <<< "$arg"
+            if [[ -n "${COMPONENT_MANIFESTS[$key]}" ]]; then
+                if [[ ! $value =~ $pattern ]]; then
+                    echo "ERROR: The value '$value' does not match the expected format 'repo-org:repo-name:branch-name:source-folder:target-folder'."
+                    continue
+                fi
+                COMPONENT_MANIFESTS["$key"]=$value
+            else
+                echo "ERROR: '$key' does not exist in COMPONENT_MANIFESTS, it will be skipped."
+                echo "Available components are: [${!COMPONENT_MANIFESTS[@]}]"
+                exit 1
+            fi
+        else
+            echo "Warning: Argument '$arg' does not follow the '--key=value' format."
+        fi
+    done
+fi
+
 # pre-cleanup local env
 rm -fr ./odh-manifests/* ./.odh-manifests-tmp/
 
-GITHUB_URL="https://github.com/"
-# update to use different git repo
-MANIFEST_ORG="opendatahub-io"
-
-# comment out below logic once we have all component manifests ready to get from source git repo
-MANIFEST_RELEASE="master"
-MANIFESTS_TARBALL_URL="${GITHUB_URL}/${MANIFEST_ORG}/odh-manifests/tarball/${MANIFEST_RELEASE}"
 mkdir -p ./.odh-manifests-tmp/ ./odh-manifests/
 wget -q -c ${MANIFESTS_TARBALL_URL} -O - | tar -zxv -C ./.odh-manifests-tmp/ --strip-components 1 > /dev/null
 cp -r ./.odh-manifests-tmp/model-mesh/ ./odh-manifests
 cp -r ./.odh-manifests-tmp/odh-model-controller/ ./odh-manifests
 cp -r ./.odh-manifests-tmp/modelmesh-monitoring/ ./odh-manifests
 cp -r ./.odh-manifests-tmp/kserve/ ./odh-manifests
-ls -lat ./odh-manifests/
 rm -rf ${MANIFEST_RELEASE}.tar.gz ./.odh-manifests-tmp/
 
-for repo_info in ${REPO_LIST[@]}; do
-    echo "Git clone below repo ${repo_info}"
-    repo_name=$( echo $repo_info | cut -d ":" -f 1 )
-    repo_branch=$( echo $repo_info | cut -d ":" -f 2 )
-    source_path=$( echo $repo_info | cut -d ":" -f 3 )
-    target_path=$( echo $repo_info | cut -d ":" -f 4 )
-    repo_url="${GITHUB_URL}/${MANIFEST_ORG}/${repo_name}.git"
+for key in "${!COMPONENT_MANIFESTS[@]}"; do
+    echo "Cloning repo ${key}: ${COMPONENT_MANIFESTS[$key]}"
+    IFS=':' read -r -a repo_info <<< "${COMPONENT_MANIFESTS[$key]}"
+
+    repo_org="${repo_info[0]}"
+    repo_name="${repo_info[1]}"
+    repo_branch="${repo_info[2]}"
+    source_path="${repo_info[3]}"
+    target_path="${repo_info[4]}"
+
+    repo_url="${GITHUB_URL}/${repo_org}/${repo_name}.git"
     rm -rf ./.${repo_name}
     git clone --depth 1 --branch ${repo_branch} ${repo_url} ./.${repo_name}
     mkdir -p ./odh-manifests/${target_path}
     cp -rf ./.${repo_name}/${source_path}/* ./odh-manifests/${target_path}
     rm -rf ./.${repo_name}
-done
+done