[WIP] AGENT-1439: split ove iso build step

AGENTS.md

@@ -26,6 +26,7 @@ dev-scripts/
 │   ├── 03_agent_build_installer.sh
 │   ├── 04_agent_prepare_release.sh
 │   ├── 05_agent_configure.sh
+│   ├── 05b_agent_build_ove_iso.sh
 │   ├── 06_agent_create_cluster.sh
 │   ├── 07_agent_add_extraworker_nodes.sh
 │   └── docs/                        # Agent-specific documentation
@@ -341,6 +342,7 @@ cd agent/
 ./03_agent_build_installer.sh
 ./04_agent_prepare_release.sh
 ./05_agent_configure.sh
+./05b_agent_build_ove_iso.sh
 ./06_agent_create_cluster.sh
 ```
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: default all agent agent_cleanup agent_build_installer agent_configure agent_create_cluster infra_only sno_bip requirements configure ironic ocp_run install_config clean ocp_cleanup ironic_cleanup host_cleanup cache_cleanup registry_cleanup proxy_cleanup workingdir_cleanup podman_cleanup bell
+.PHONY: default all agent agent_cleanup agent_build_installer agent_build_ove_iso agent_configure agent_create_cluster infra_only sno_bip requirements configure ironic ocp_run install_config clean ocp_cleanup ironic_cleanup host_cleanup cache_cleanup registry_cleanup proxy_cleanup workingdir_cleanup podman_cleanup bell
 default: requirements configure build_installer ironic install_config ocp_run bell
 
 all: default
@@ -7,7 +7,7 @@ all: default
 assisted: assisted_deployment bell
 
 # Deploy cluster with agent installer flow
-agent: agent_requirements requirements configure agent_build_installer agent_prepare_release agent_configure agent_create_cluster 
+agent: agent_requirements requirements configure agent_build_installer agent_prepare_release agent_configure agent_build_ove_iso agent_create_cluster
 
 # Deploy cluster with agent installer flow and adds nodes after initial install
 # Requires at least 1 extra worker node to be configured with disk size at least 100Gß
@@ -41,6 +41,9 @@ agent_prepare_release:
 agent_configure:
 	./agent/05_agent_configure.sh
 
+agent_build_ove_iso:
+	./agent/05b_agent_build_ove_iso.sh
+
 agent_create_cluster:
 	./agent/06_agent_create_cluster.sh
 

agent/iso_no_registry.sh → agent/05b_agent_build_ove_iso.sh

@@ -1,16 +1,25 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -euxo pipefail
 
-# OVE (OpenShift Virtualization Edition) ISO building utilities
-# Functions for creating agent ISOs without embedded registry
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+ARCH=$(uname -m)
+
+LOGDIR="${SCRIPTDIR}/logs"
+source "$SCRIPTDIR/logging.sh"
+source "$SCRIPTDIR/common.sh"
+source "$SCRIPTDIR/network.sh"
+source "$SCRIPTDIR/utils.sh"
+source "$SCRIPTDIR/validation.sh"
+source "$SCRIPTDIR/release_info.sh"
+source "$SCRIPTDIR/agent/common.sh"
+
+early_deploy_validation
 
-# Check if using a custom registry (not upstream quay.io or CI registry)
 function is_custom_registry() {
   [[ ! "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}" =~ quay\.io ]] && \
   [[ ! "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}" =~ registry\.ci\.openshift\.org ]]
 }
 
-# Determine release image URL based on mirror configuration
 function get_release_image_url() {
   if [[ "${MIRROR_IMAGES}" == "true" ]] && [[ -n "$(get_repo_overrides)" ]]; then
     echo "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}"
@@ -19,7 +28,6 @@ function get_release_image_url() {
   fi
 }
 
-# Build OVE ISO using script method
 function build_ove_iso_script() {
   local asset_dir=$1
   local release_image_url=$2
@@ -38,31 +46,25 @@ function build_ove_iso_script() {
     ${registry_cert_arg}
 }
 
-# Build OVE ISO using container method
 function build_ove_iso_container() {
   local asset_dir=$1
   local release_image_url=$2
 
-  # Build ISO in container
   make build-ove-iso-container \
     PULL_SECRET_FILE="${PULL_SECRET_FILE}" \
     RELEASE_IMAGE_URL="${release_image_url}" \
     ARCH="${ARCH}"
 
-  # Extract ISO from container
   ./hack/iso-from-container.sh
 
-  # Move to asset directory
   local iso_name="agent-ove.${ARCH}.iso"
   echo "Moving ${iso_name} to ${asset_dir}"
   mv "./output-iso/${iso_name}" "${asset_dir}"
 }
 
-# Create agent ISO without registry (OVE ISO)
 function create_agent_iso_no_registry() {
   local asset_dir=${1}
 
-  # Update release_info.json as its needed by CI tests
   save_release_info "${OPENSHIFT_RELEASE_IMAGE}" "${OCP_DIR}"
 
   local src_dir
@@ -79,26 +81,22 @@ function create_agent_iso_no_registry() {
   pushd .
   cd "${src_dir}"
 
-  # Determine release image URL
   local release_image_url
   release_image_url=$(get_release_image_url)
   echo "build_ove_iso will use release image ${release_image_url}"
 
-  # Prepare mirror and certificate arguments for script build method
   local mirror_path_arg=""
   local registry_cert_arg=""
 
   if [[ "${MIRROR_IMAGES}" == "true" ]]; then
     echo "Using pre-mirrored images from ${REGISTRY_DIR}"
     mirror_path_arg="--mirror-path ${REGISTRY_DIR}"
 
-    # Add registry certificate if using custom registry
     if is_custom_registry && [[ -f "${REGISTRY_DIR}/certs/${REGISTRY_CRT}" ]]; then
       registry_cert_arg="--registry-cert ${REGISTRY_DIR}/certs/${REGISTRY_CRT}"
     fi
   fi
 
-  # Build OVE ISO using selected method
   if [[ "${AGENT_ISO_NO_REGISTRY_BUILD_METHOD}" == "script" ]]; then
     build_ove_iso_script "${asset_dir}" "${release_image_url}" "${mirror_path_arg}" "${registry_cert_arg}"
   else
@@ -109,25 +107,58 @@ function create_agent_iso_no_registry() {
   popd
 }
 
-# Deletes all files and directories under asset_dir
-# example, ocp/ostest/iso_builder/4.19.* 
-# except the final generated ISO file (agent-ove.${ARCH}.iso),
-# to free up disk space while preserving the built artifact.
-# Note: This optional cleanup is relevant only when the
-# AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV is set as as true, 
+function assert_agent_no_registry_iso_size(){
+  agent_iso_no_registry=$(get_agent_iso_no_registry)
+  iso_size=$(stat -c%s "$agent_iso_no_registry")
+
+  iso_size_limit=$((AGENT_OVE_ISO_SIZE * 1024 * 1024 * 1024))
+
+  if (( iso_size > iso_size_limit )); then
+    echo "Error: OVE ISO size of $agent_iso_no_registry is ${iso_size}, which exceeds the ${AGENT_OVE_ISO_SIZE}GB limit."
+    exit 1
+  fi
+}
+
 function cleanup_diskspace_agent_iso_noregistry() {
- local asset_dir=${1%/}  # Remove trailing slash if present
+  local asset_dir=${1%/}
 
-  # Iterate over all versioned directories
   for dir in "$asset_dir"/[0-9]*.[0-9]*.*; do
     [ -d "$dir" ] || continue
 
     echo "Cleaning up directory: $dir"
 
-    # Delete all files and symlinks except the agent-ove ISO
-    sudo find "$dir" \( -type f -o -type l \) ! -name "agent-ove.${ARCH}.iso" -print -delete
+    sudo find "$dir" \( -type f -o -type l \) ! -name "agent-ove.${ARCH}.iso" -delete
 
-    # Remove any empty directories left behind
-    sudo find "$dir" -type d -empty -print -delete
+    sudo find "$dir" -type d -empty -delete
   done
 }
+
+# Main
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" != "ISO_NO_REGISTRY" ]]; then
+    echo "Skipping OVE ISO build: AGENT_E2E_TEST_BOOT_MODE=${AGENT_E2E_TEST_BOOT_MODE}"
+    exit 0
+fi
+
+if agent_iso=$(get_agent_iso_no_registry 2>/dev/null); then
+    echo "OVE ISO already exists at ${agent_iso}, skipping build"
+    exit 0
+fi
+
+asset_dir=${AGENT_OVE_ISO_PATH}/iso_builder
+mkdir -p "${asset_dir}"
+
+create_agent_iso_no_registry "${asset_dir}"
+
+assert_agent_no_registry_iso_size
+
+if [[ "$AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV" == "true" ]]; then
+    cleanup_diskspace_agent_iso_noregistry "${asset_dir}"
+fi
+
+if [[ "${MIRROR_IMAGES}" == "true" ]]; then
+    echo "Cleaning up registry data at ${REGISTRY_DIR} to save disk space"
+    sudo rm -rf "${REGISTRY_DIR}/data"
+    echo "Registry data cleanup complete"
+fi
+
+echo "OVE ISO build complete"

agent/06_agent_create_cluster.sh

@@ -19,7 +19,6 @@ source "$SCRIPTDIR/validation.sh"
 source "$SCRIPTDIR/release_info.sh"
 source "$SCRIPTDIR/agent/common.sh"
 source "$SCRIPTDIR/agent/iscsi_utils.sh"
-source "$SCRIPTDIR/agent/iso_no_registry.sh"
 
 early_deploy_validation
 
@@ -90,19 +89,6 @@ function create_config_image() {
 }
 
 
-function assert_agent_no_registry_iso_size(){
-  agent_iso_no_registry=$(get_agent_iso_no_registry)
-  iso_size=$(stat -c%s "$agent_iso_no_registry")
-
-  # With 4.19 tech preview, the expected ISO size is approximately 36GB
-  iso_size_limit=$((AGENT_OVE_ISO_SIZE * 1024 * 1024 * 1024))
-
-  if (( iso_size > iso_size_limit )); then
-    echo "Error: OVE ISO size of $agent_iso_no_registry is ${iso_size}, which exceeds the ${AGENT_OVE_ISO_SIZE}GB limit."
-    exit 1
-  fi
-}
-
 function set_device_config_image() {
 
     for (( n=0; n<${2}; n++ ))
@@ -127,18 +113,6 @@ function get_agent_iso() {
     echo "${agent_iso}"
 }
 
-function get_agent_iso_no_registry() {
-    local base_dir=$SCRIPTDIR/$OCP_DIR
-    local iso_name="agent-ove.${ARCH}.iso"
-    local agent_iso_no_registry=$
-    agent_iso_no_registry=$(find "$base_dir" -type f -name "$iso_name" 2>/dev/null | head -n 1)
-    if [ -z "$agent_iso_no_registry" ]; then
-      echo "Error: No agent OVE ISO found matching ${iso_name} in ${base_dir}" >&2
-      exit 1
-    fi
-    echo "${agent_iso_no_registry}"
-}
-
 function attach_agent_iso() {
 
     set_file_acl
@@ -677,25 +651,8 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
     sudo rm -rf "${OCP_DIR}/temp"
     ;;
   "ISO_NO_REGISTRY" )
-    # Build an (OVE) image which does not need registry setup 
-    # Run a script from agent-installer-utils which internally uses openshift-appliance
-    asset_dir=$SCRIPTDIR/$OCP_DIR/iso_builder
-    mkdir -p "${asset_dir}"
-    create_agent_iso_no_registry "${asset_dir}"
-
-    assert_agent_no_registry_iso_size
-
-    if [[ "$AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV" == "true" ]]; then
-      # reclaim disk space by deleting unwanted cache, other files
-      cleanup_diskspace_agent_iso_noregistry "${asset_dir}"
-    fi
-
-    # Clean up registry data to save disk space after ISO is created
-    if [[ "${MIRROR_IMAGES}" == "true" ]]; then
-      echo "Cleaning up registry data at ${REGISTRY_DIR} to save disk space"
-      sudo rm -rf "${REGISTRY_DIR}/data"
-      echo "Registry data cleanup complete"
-    fi
+    # OVE ISO must already exist, built by the agent_build_ove_iso step
+    echo "Using pre-built OVE ISO: $(get_agent_iso_no_registry)"
 
     attach_agent_iso_no_registry master "$NUM_MASTERS"
     attach_agent_iso_no_registry worker "$NUM_WORKERS"

agent/README.md

@@ -34,7 +34,8 @@ Then run the `agent` target:
 | 02 | configure | Setup the network and VMs as per the specified configuration | no |
 | 03 | agent_build_installer | Builds (or extract from the release payload) the openshift installer | yes |
 | 04 | agent_configure | Further network customization and agent manifests creation (stored in `OCP_DIR`)  | yes |
-| 05 | agent_create_cluster | Generates the agent image using openshift installer and boots the VM | yes |
+| 05b | agent_build_ove_iso | Builds the OVE ISO for ISO_NO_REGISTRY mode (no-op for other modes) | yes |
+| 06 | agent_create_cluster | Generates the agent image using openshift installer and boots the VM | yes |
 | - | agent_cleanup | Deletes the agent manifests and images | yes | 
 
 # Agent artifacts

agent/common.sh

@@ -127,3 +127,15 @@ function getAgentISOBuilderImage() {
 function get_repo_overrides() {
     env | grep '_LOCAL_REPO=' | grep -o '^[^=]*' || true
 }
+
+function get_agent_iso_no_registry() {
+    local search_dir=${AGENT_OVE_ISO_PATH}
+    local iso_name="agent-ove.${ARCH}.iso"
+    local agent_iso_no_registry
+    agent_iso_no_registry=$(find "$search_dir" -type f -name "$iso_name" 2>/dev/null | head -n 1)
+    if [ -z "$agent_iso_no_registry" ]; then
+      echo "Error: No agent OVE ISO found matching ${iso_name} in ${search_dir}" >&2
+      exit 1
+    fi
+    echo "${agent_iso_no_registry}"
+}

common.sh

@@ -452,6 +452,7 @@ export AGENT_E2E_TEST_BOOT_MODE=${AGENT_E2E_TEST_BOOT_MODE:-"ISO"}
 export AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV=${AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV:-"false"}
 export AGENT_RENDEZVOUS_NODE_HOSTNAME=${AGENT_RENDEZVOUS_NODE_HOSTNAME:-${CLUSTER_NAME}_master_0}
 export AGENT_OVE_ISO_SIZE=${AGENT_OVE_ISO_SIZE:-70}
+export AGENT_OVE_ISO_PATH=${AGENT_OVE_ISO_PATH:-"$SCRIPTDIR/$OCP_DIR"}
 
 # HTTP boot server port used by the agent installer for PXE and minimal ISO
 # Needed to be defined here since it's required also by the shared step 02_configure_host.sh

config_example.sh

@@ -933,6 +933,14 @@ set -x
 # The script method is recommended for local development as it allows faster iteration and easier debugging.
 # export AGENT_ISO_NO_REGISTRY_BUILD_METHOD=script
 
+# AGENT_OVE_ISO_PATH specifies the directory where the OVE ISO is built and searched for
+# when AGENT_E2E_TEST_BOOT_MODE is set to ISO_NO_REGISTRY.
+# Defaults to "$SCRIPTDIR/$OCP_DIR" (e.g. /path/to/dev-scripts/ocp/ostest).
+# In CI, this can be set to a shared filesystem path so that the ISO build step
+# (make agent_build_ove_iso) and the cluster deployment step (make agent) can run
+# as separate CI steps with independent timeouts, even from different working directories.
+# export AGENT_OVE_ISO_PATH=/shared/ove
+
 # Specifies the hostname of the node that should be identified and set as the rendezvous node 
 # during the OVE cluster installation process. This node acts as the bootstrap node in the cluster.
 # Accepts only master nodes.