From 5ce267c2bba41ef67cdfe0cbed98a9ef5b6ecbfa Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 22 May 2026 18:58:57 -0400 Subject: [PATCH] feat(ci): add _build_weblog_images option to skip weblog build jobs Introduces a build_weblog_images parameter that controls whether weblogs get dedicated build jobs. When false (nodejs without base images label), weblogs are rebuilt inline during the run job instead. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../workflows/compute-workflow-parameters.yml | 6 +++ .github/workflows/system-tests.yml | 1 + .../scripts/ci_orchestrators/workflow_data.py | 42 +++++++++++++++---- utils/scripts/compute-workflow-parameters.py | 4 ++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index 499ce90a030..21664067d70 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -52,6 +52,11 @@ on: default: '' required: false type: string + _build_weblog_images: + description: "When true, weblogs get a build job to save artifacts. When false, parallel_weblogs is empty and run jobs rebuild from Docker Hub." + default: true + required: false + type: boolean # Map the workflow outputs to job outputs outputs: @@ -137,6 +142,7 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ + --build-weblog-images "${{ inputs._build_weblog_images }}" \ --output $GITHUB_OUTPUT - name: log run: | diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 23c31c6d4c9..0e6af9f9b67 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -168,6 +168,7 @@ jobs: binaries_artifact: ${{ inputs.binaries_artifact }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }} + _build_weblog_images: ${{ inputs.library != 'nodejs' || inputs._build_nodejs_base_images }} parametric: needs: diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 9ff5dc2ee71..e28f81bb22b 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -201,6 +201,7 @@ class Weblog: name: str require_build: bool artifact_name: str + build_in_run: bool = False def serialize(self) -> dict: return {"name": self.name, "artifact_name": self.artifact_name} @@ -232,7 +233,7 @@ def serialize(self) -> dict: "runs_on": "ubuntu-latest", "library": self.library, "weblog": self.weblog.name, - "weblog_build_required": self.weblog.require_build, + "weblog_build_required": self.weblog.require_build or self.weblog.build_in_run, "weblog_instance": self.weblog_instance, "scenarios": sorted(self.scenarios), "expected_job_time": self.expected_job_time + self.build_time, @@ -279,7 +280,13 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J def _get_endtoend_weblogs( - library: str, weblogs_filter: list[str], unique_id: str, ci_environment: str, binaries_artifact: str + library: str, + weblogs_filter: list[str], + unique_id: str, + ci_environment: str, + binaries_artifact: str, + *, + build_weblog_images: bool = True, ) -> list[Weblog]: result: list[Weblog] = [] @@ -310,13 +317,23 @@ def _get_endtoend_weblogs( for name in names: if name not in integration_frameworks_weblogs: - result.append( - Weblog( - name=name, - require_build=True, - artifact_name=f"binaries_{ci_environment}_{library}_{name}_{unique_id}", + if not build_weblog_images: + result.append( + Weblog( + name=name, + require_build=False, + build_in_run=True, + artifact_name=binaries_artifact, + ) + ) + else: + result.append( + Weblog( + name=name, + require_build=True, + artifact_name=f"binaries_{ci_environment}_{library}_{name}_{unique_id}", + ) ) - ) else: for version in integration_frameworks_weblogs[name]: result.append( @@ -343,6 +360,8 @@ def get_endtoend_definitions( maximum_parallel_jobs: int, unique_id: str, binaries_artifact: str, + *, + build_weblog_images: bool = True, ) -> dict: scenarios = scenario_map["endtoend"] @@ -352,7 +371,12 @@ def get_endtoend_definitions( # get the list of end-to-end weblogs for the given library weblogs: list[Weblog] = _get_endtoend_weblogs( - library, weblogs_filter, ci_environment=ci_environment, unique_id=unique_id, binaries_artifact=binaries_artifact + library, + weblogs_filter, + ci_environment=ci_environment, + unique_id=unique_id, + binaries_artifact=binaries_artifact, + build_weblog_images=build_weblog_images, ) # check that jobs can be splitted diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 9048ef66a94..c9d5bf245aa 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,6 +43,7 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, + build_weblog_images: bool = True, ): # this data struture is a dict where: # the key is the workflow identifier @@ -87,6 +88,7 @@ def __init__( maximum_parallel_jobs=256, unique_id=self.unique_id, binaries_artifact=self.binaries_artifact, + build_weblog_images=build_weblog_images, ) self.data["parametric"] = { @@ -284,6 +286,7 @@ def _get_workflow_map( parser.add_argument( "--system-tests-dev-mode", type=str, help="true if running in system-tests CI, with the dev mode", default="" ) + parser.add_argument("--build-weblog-images", type=str, help="When true, weblogs get a build job", default="true") parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) args = parser.parse_args() @@ -304,4 +307,5 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, + build_weblog_images=args.build_weblog_images != "false", ).export(export_format=args.format, output=args.output)