agent: introduce openshift-appliance flow

agent/06_agent_create_cluster.sh

@@ -82,12 +82,16 @@ function set_device_config_image() {
     done
 }
 
+function set_file_acl() {
+  # This is required to allow qemu opening the disk image
+  if [ "${OPENSHIFT_CI}" == true ]; then
+    setfacl -m u:qemu:rx /root
+  fi
+}
+
 function attach_agent_iso() {
 
-    # This is required to allow qemu opening the disk image
-    if [ "${OPENSHIFT_CI}" == true ]; then
-      setfacl -m u:qemu:rx /root
-    fi
+    set_file_acl
 
     local agent_iso="${OCP_DIR}/agent.$(uname -p).iso"
     if [ ! -f "${agent_iso}" -a -f "${OCP_DIR}/agent.iso" ]; then
@@ -112,6 +116,35 @@ function attach_agent_iso() {
 
 }
 
+function attach_appliance_diskimage() {
+    set_file_acl
+
+    local config_image_drive="sdd"
+    local appliance_disk_image="${OCP_DIR}/appliance.raw"
+
+    # Create the config ISO
+    mkdir -p ${config_image_dir}
+    cp ${asset_dir}/*.yaml ${config_image_dir}
+    create_config_image
+
+    for (( n=0; n<${2}; n++ ))
+    do
+        name=${CLUSTER_NAME}_${1}_${n}
+        disk_image=${appliance_disk_image}_${1}_${n}
+
+        # Every node needs a copy of the appliance disk image
+        sudo cp "${appliance_disk_image}" "${disk_image}"
+
+        # Attach the appliance disk image and the config ISO 
+        sudo virt-xml ${name} --remove-device --disk 1
+        sudo virt-xml ${name} --add-device --disk "${disk_image}",device=disk,target.dev=sda
+        sudo virt-xml ${name} --add-device --disk "${config_image_dir}/agentconfig.noarch.iso",device=cdrom,target.dev=${config_image_drive}
+
+        # Boot machine from the appliance disk image
+        sudo virt-xml ${name} --edit target=sda --disk="boot_order=1" --start
+    done
+}
+
 function get_node0_ip() {
   node0_name=$(printf ${MASTER_HOSTNAME_FORMAT} 0)
   node0_ip=$(sudo virsh net-dumpxml ostestbm | xmllint --xpath "string(//dns[*]/host/hostname[. = '${node0_name}']/../@ip)" -)
@@ -178,7 +211,7 @@ function enable_assisted_service_ui() {
 function wait_for_cluster_ready() {
   local openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
   local dir="${OCP_DIR}"
-  if [[ "${AGENT_USE_APPLIANCE_MODEL}" == true ]]; then
+  if [[ "${AGENT_USE_APPLIANCE_MODEL}" == true || "${AGENT_E2E_TEST_BOOT_MODE}" == "DISKIMAGE" ]]; then
      dir="${config_image_dir}"
   fi
   if ! "${openshift_install}" --dir="${dir}" --log-level=debug agent wait-for bootstrap-complete; then
@@ -294,6 +327,14 @@ function agent_pxe_boot() {
       done
 }
 
+function create_appliance() {
+    local asset_dir="$(realpath "${1}")"
+
+    # Build appliance with `debug-base-ignition` flag for using the custom openshift-install
+    # binary from assets directory.
+    sudo podman run -it --rm --pull newer --privileged --net=host -v ${asset_dir}:/assets:Z ${APPLIANCE_IMAGE} build --debug-base-ignition
+}
+
 asset_dir="${1:-${OCP_DIR}}"
 config_image_dir="${1:-${OCP_DIR}/configimage}"
 openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
@@ -315,7 +356,16 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
 
     agent_pxe_boot master $NUM_MASTERS
     agent_pxe_boot worker $NUM_WORKERS
-    ;;    
+    ;;
+
+  "DISKIMAGE" )
+    # Build disk image using openshift-appliance
+    create_appliance ${asset_dir}
+
+    # Attach the diskimage to nodes
+    attach_appliance_diskimage master $NUM_MASTERS
+    attach_appliance_diskimage worker $NUM_WORKERS
+    ;;
 esac
 
 if [ ! -z "${AGENT_TEST_CASES:-}" ]; then

agent/cleanup.sh

@@ -20,6 +20,11 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
     sudo pkill agentpxeserver || true
     rm -rf ${WORKING_DIR}/pxe
     ;;
+  "DISKIMAGE" )
+    sudo rm -rf "${OCP_DIR}/cache"
+    sudo rm -rf "${OCP_DIR}/temp"
+    sudo podman rmi -f ${APPLIANCE_IMAGE} || true
+    ;;
 esac
 
 sudo podman rm -f extlb || true

agent/common.sh

@@ -11,6 +11,10 @@ export AGENT_USE_APPLIANCE_MODEL=${AGENT_USE_APPLIANCE_MODEL:-"false"}
 export AGENT_APPLIANCE_HOTPLUG=${AGENT_APPLIANCE_HOTPLUG:-"false"}
 export AGENT_PLATFORM_TYPE=${AGENT_PLATFORM_TYPE:-"baremetal"}
 
+# Image reference for OpenShift-based Appliance Builder.
+# See: https://github.com/openshift/appliance
+export APPLIANCE_IMAGE=${APPLIANCE_IMAGE:-"quay.io/edge-infrastructure/openshift-appliance:latest"}
+
 # Override command name in case of extraction
 export OPENSHIFT_INSTALLER_CMD="openshift-install"
 

agent/roles/manifests/tasks/appliance.yml

@@ -0,0 +1,4 @@
+- name: write the appliance-config.yaml 
+  template:
+    src: "appliance-config_yaml.j2"
+    dest: "{{ install_path }}/appliance-config.yaml"

agent/roles/manifests/tasks/main.yml

@@ -25,3 +25,7 @@
 - name: Create ZTP based manifests
   import_tasks: ztp.yml
   when: agent_use_ztp_manifests == 'true'
+
+- name: Create appliance-config manifests
+  import_tasks: appliance.yml
+  when: boot_mode == 'DISKIMAGE'

agent/roles/manifests/templates/agent-config_yaml.j2

@@ -7,7 +7,7 @@
 {% set test_cases = agent_test_cases.split(',') %}
 apiVersion: v1alpha1
 metadata:
-  name: {{ cluster_name }} 
+  name: {{ cluster_name }}
   namespace: {{ cluster_namespace }}
 rendezvousIP: {{ ips[0] }} 
 {% if boot_mode == "PXE" %}

agent/roles/manifests/templates/appliance-config_yaml.j2

@@ -0,0 +1,11 @@
+apiVersion: v1beta1
+kind: ApplianceConfig
+ocpRelease:
+  version: {{ version }}
+  channel: candidate
+  cpuArchitecture: {{ ansible_architecture }}
+diskSizeGB: 200
+pullSecret: {{ pull_secret_contents }}
+sshKey: {{ ssh_pub_key }}
+imageRegistry:
+  uri: quay.io/libpod/registry:2.8

agent/roles/manifests/templates/install-config_yaml.j2

@@ -12,7 +12,9 @@ controlPlane:
   replicas: {{ num_masters }} 
 metadata:
   name: {{ cluster_name }} 
+{% if boot_mode != "DISKIMAGE" %}
   namespace: {{ cluster_namespace }}
+{% endif %}
 networking:
 {% if ip_stack == "v4" %}
   clusterNetwork:

common.sh

@@ -457,10 +457,15 @@ if [[ ! -z ${AGENT_E2E_TEST_SCENARIO} ]]; then
 fi
 
 if [[ ! -z ${AGENT_E2E_TEST_BOOT_MODE} ]]; then
-  if [[ $AGENT_E2E_TEST_BOOT_MODE != "ISO" && $AGENT_E2E_TEST_BOOT_MODE != "PXE" ]]; then
-      printf "Found invalid value \"$AGENT_E2E_TEST_BOOT_MODE\" for AGENT_E2E_TEST_BOOT_MODE. Supported values: ISO (default), PXE."
+  case "$AGENT_E2E_TEST_BOOT_MODE" in
+    "ISO" | "PXE" | "DISKIMAGE")
+      # Valid value
+      ;;
+    *)
+      printf "Found invalid value \"$AGENT_E2E_TEST_BOOT_MODE\" for AGENT_E2E_TEST_BOOT_MODE. Supported values: ISO (default), PXE, DISKIMAGE."
       exit 1
-  fi
+      ;;
+  esac
 fi
 
 # For IPv6 (default case) mirror images are used since quay doesn't support IPv6

config_example.sh

@@ -694,6 +694,7 @@ set -x
 # When set, the code internally sets the boot mode for the agents.
 # This config variable is used only by the agent based installer and is optional.
 # The default value for AGENT_E2E_TEST_BOOT_MODE is 'ISO'.
+# For the openshift-appliance flow, set the boot mode to 'DISKIMAGE'.
 # AGENT_E2E_TEST_BOOT_MODE=PXE
 
 # Uncomment and set the following value to "true" to enable a test scenario
@@ -733,3 +734,9 @@ set -x
 # haproxy is deployed as the load balancer, and the appropriate DNS records
 # are created.
 # export AGENT_PLATFORM_TYPE=none
+
+# Image reference for OpenShift-based Appliance Builder.
+# See: https://github.com/openshift/appliance
+# Default: "quay.io/edge-infrastructure/openshift-appliance:latest"
+#
+# export APPLIANCE_IMAGE="quay.io/edge-infrastructure/openshift-appliance:latest"