diff --git a/.gitattributes b/.gitattributes index 9e7da37ad..7590623c3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,11 +4,15 @@ doc/changes/changelog.md linguist-generated=true .github/actions/python-environment/ext/get_poetry.py linguist-generated=true # PTB GitHub Workflows (CI/CD & Publishing) -.github/workflows/build-and-publish.yml linguist-generated=true -.github/workflows/cd.yml linguist-generated=true -.github/workflows/check-release-tag.yml linguist-generated=true -.github/workflows/ci.yml linguist-generated=true -.github/workflows/gh-pages.yml linguist-generated=true -.github/workflows/matrix-*.yml linguist-generated=true -.github/workflows/pr-merge.yml linguist-generated=true -.github/workflows/report.yml linguist-generated=true +.github/workflows/build-and-publish.yml linguist-generated=true +.github/workflows/cd.yml linguist-generated=true +.github/workflows/check-release-tag.yml linguist-generated=true +.github/workflows/checks.yml linguist-generated=true +.github/workflows/ci.yml linguist-generated=true +.github/workflows/fast-tests.yml linguist-generated=true +.github/workflows/gh-pages.yml linguist-generated=true +.github/workflows/matrix-*.yml linguist-generated=true +.github/workflows/merge-gate.yml linguist-generated=true +.github/workflows/periodic-validation.yml linguist-generated=true +.github/workflows/pr-merge.yml linguist-generated=true +.github/workflows/report.yml linguist-generated=true diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 91277c4fa..0afc4c9cd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -187,59 +187,3 @@ jobs: - name: Build Package id: build-package run: poetry run -- nox -s package:check - - lint-imports: - name: Lint Imports - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Lint Imports - id: lint-imports - run: poetry run -- nox -s lint:import - - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "ubuntu-24.04" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run Unit Tests - id: run-unit-tests - run: poetry run -- nox -s test:unit -- --coverage - - - name: Upload Artifacts - id: upload-artifacts - uses: actions/upload-artifact@v7 - with: - name: coverage-python${{ matrix.python-versions }}-fast - path: .coverage - include-hidden-files: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82c8388b1..b6171e32b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,26 +3,16 @@ name: CI on: pull_request: types: [opened, synchronize, reopened] - schedule: - # At 00:00 on every 7th day-of-month from 1 through 31. (https://crontab.guru) - - cron: "0 0 1/7 * *" jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - with: - root-event: ${{ github.event_name }} secrets: inherit permissions: contents: read report: - # Job merge-gate requires manual approval for running the slow checks. If - # current workflow ci.yml is triggered by schedule, there is no manual - # interaction, manual approval will never be given, slow checks will not - # be executed, merge-gate will never terminate, and the report will never - # be called. name: Report needs: - merge-gate diff --git a/.github/workflows/fast-tests-extension.yml b/.github/workflows/fast-tests-extension.yml new file mode 100644 index 000000000..ed1c038ba --- /dev/null +++ b/.github/workflows/fast-tests-extension.yml @@ -0,0 +1,26 @@ +name: Fast-Tests-Extension + +on: + workflow_call: + +jobs: + lint-imports: + name: Lint Imports + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v7 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Lint Imports + id: lint-imports + run: poetry run -- nox -s lint:import diff --git a/.github/workflows/fast-tests.yml b/.github/workflows/fast-tests.yml new file mode 100644 index 000000000..28a5a6954 --- /dev/null +++ b/.github/workflows/fast-tests.yml @@ -0,0 +1,45 @@ +name: Fast-Tests + +on: + workflow_call: + +jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v7 + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + + - name: Run Unit Tests + id: run-unit-tests + run: poetry run -- nox -s test:unit -- --coverage + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python${{ matrix.python-versions }}-fast + path: .coverage + include-hidden-files: true + + fast-tests-extension: + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: read diff --git a/.github/workflows/merge-gate-extension.yml b/.github/workflows/merge-gate-extension.yml new file mode 100644 index 000000000..1e05a6ffb --- /dev/null +++ b/.github/workflows/merge-gate-extension.yml @@ -0,0 +1,11 @@ +name: Slow-Checks-Extension + +on: + workflow_call: + +jobs: + test-python-environment: + name: Test python-environment Action + uses: ./.github/workflows/test-python-environment.yml + permissions: + contents: read diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 24200ed93..00c29af9e 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -2,12 +2,6 @@ name: Merge-Gate on: workflow_call: - inputs: - root-event: - description: GitHub event triggering the root workflow ci.yml - required: false - type: string - default: unknown jobs: run-fast-checks: @@ -16,10 +10,17 @@ jobs: permissions: contents: read + run-fast-tests: + name: Fast Tests + uses: ./.github/workflows/fast-tests.yml + permissions: + contents: read + fast-report: name: Fast Report needs: - run-fast-checks + - run-fast-tests uses: ./.github/workflows/report.yml secrets: inherit permissions: @@ -27,7 +28,6 @@ jobs: approve-run-slow-tests: name: Approve Running Slow Tests? - if: ${{ inputs.root-event != 'schedule' }} runs-on: "ubuntu-24.04" permissions: contents: read @@ -48,11 +48,9 @@ jobs: permissions: contents: read - test-python-environment: - name: Test python-environment Action - needs: - - approve-run-slow-tests - uses: ./.github/workflows/test-python-environment.yml + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + secrets: inherit permissions: contents: read @@ -67,7 +65,7 @@ jobs: needs: - run-fast-checks - run-slow-checks - - test-python-environment + - merge-gate-extension # To prevent accidentally merges, this step is required. For more details # see: https://github.com/exasol/python-toolbox/issues/563 steps: diff --git a/.github/workflows/periodic-validation.yml b/.github/workflows/periodic-validation.yml new file mode 100644 index 000000000..219de1bd6 --- /dev/null +++ b/.github/workflows/periodic-validation.yml @@ -0,0 +1,37 @@ +name: Periodic-Validation + +on: + schedule: + # At 00:00 on Saturday. (https://crontab.guru) + - cron: "0 0 * * 6" + +jobs: + run-fast-checks: + name: Fast Checks + uses: ./.github/workflows/checks.yml + permissions: + contents: read + + run-fast-tests: + name: Fast Tests + uses: ./.github/workflows/fast-tests.yml + permissions: + contents: read + + run-slow-checks: + name: Slow Checks + uses: ./.github/workflows/slow-checks.yml + secrets: inherit + permissions: + contents: read + + report: + name: Report + needs: + - run-fast-checks + - run-fast-tests + - run-slow-checks + uses: ./.github/workflows/report.yml + secrets: inherit + permissions: + contents: read diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml index 482647ee8..861e45e8a 100644 --- a/.github/workflows/pr-merge.yml +++ b/.github/workflows/pr-merge.yml @@ -7,14 +7,6 @@ on: - 'master' jobs: - # This job can be removed if certain preconditions are met. See - # https://exasol.github.io/python-toolbox/user_guide/workflows.html#pr-merge-workflow - run-fast-checks: - name: Run Fast Checks - uses: ./.github/workflows/checks.yml - permissions: - contents: read - publish-docs: name: Publish Documentation uses: ./.github/workflows/gh-pages.yml @@ -22,11 +14,3 @@ jobs: contents: read pages: write id-token: write - - report: - needs: - - run-fast-checks - uses: ./.github/workflows/report.yml - secrets: inherit - permissions: - contents: read diff --git a/.workflow-patcher.yml b/.workflow-patcher.yml index 19dad4ab2..270c0dff2 100644 --- a/.workflow-patcher.yml +++ b/.workflow-patcher.yml @@ -1,5 +1,5 @@ workflows: - - name: checks + - name: fast-tests step_customizations: - action: REPLACE job: run-unit-tests diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 07091ef24..a6849dee8 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,6 +2,17 @@ ## Summary +In this major release, several modifications were made to the PTB's workflow templates: + +* The periodic run which was previously executed in the `ci.yml` has been moved to its +own `periodic-validation.yml` and will run weekly. This also has been modified to +run the `slow-checks.yml` so that more complete linting and coverage information is +sent to Sonar. +* The unit tests job has been moved from `checks.yml` to its own `fast-tests.yml` file. +* Workflow extensions were added to `fast-tests` and `merge-gate`. This allows users to +add custom `fast-tests-extension.yml` and `merge-gate-extension.yml` files. For more +details, check out LINK TO BE ADDED (DOES NOT EXIST YET). + ## Bugfix * #563: Fixed merge-gate to prevent auto-merges from happening when integration tests failed @@ -9,3 +20,7 @@ ## Feature * #829: Extended removing a job from a workflow to also remove it from the `needs` of another job +* #825: Created two workflows by splitting up previous ones: + * Moved the periodic jobs in `ci.yml` to its own `periodic-validation.yml` + * Moved the unit tests job in `checks.yml` to its own `fast-tests.yml` +* #730: Added workflow extensions to `fast-tests` and `merge-gate` diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 366918dc2..ed7d95161 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -32,6 +32,13 @@ workflows from the templates. Workflows --------- +The PTB has two categories of workflows: + #. those maintained by the PTB, which can be modified using the :ref:`workflow_patcher`. + #. those which extend the PTB-provided workflows and are maintained by the project (not the PTB). + +Maintained by the PTB +^^^^^^^^^^^^^^^^^^^^^ + .. list-table:: :widths: 25 25 50 :header-rows: 1 @@ -53,12 +60,15 @@ Workflows * - ``checks.yml`` - Workflow call - Executes many small & fast checks: builds documentation and validates - cross-references (AKA. "links") to be valid,runs various linters - (security, type checks, etc.), and unit tests. + cross-references (AKA. "links") to be valid, and runs various linters + (security, type checks, etc.). * - ``ci.yml`` - - Pull request and monthly + - Pull request - Executes the continuous integration suite by calling ``merge-gate.yml`` and ``report.yml``. See :ref:`ci_yml` for a graph of workflow calls. + * - ``fast-tests.yml`` + - Workflow call + - Executes unit tests. * - ``gh-pages.yml`` - Workflow call - Builds the documentation and deploys it to GitHub Pages. @@ -79,10 +89,13 @@ Workflows - Acts as a final status check (gatekeeper) to ensure all required CI steps have passed before allowing to merge the branch of your pull request to the default branch of the repository. e.g. ``main``. + * - ``periodic-validation.yml`` + - weekly + - Acts as a periodic validator that critical checks and tests are working as + expected. See :ref:`periodic_validation_yml` for a graph of workflow calls. * - ``pr-merge.yml`` - Push to main - - Runs ``checks.yml``, ``gh-pages.yml``, and ``report.yml``. See - :ref:`pr_merge_yml` for a graph of called workflows. + - Runs ``gh-pages.yml``. See :ref:`pr_merge_yml` for a graph of called workflows. * - ``report.yml`` - Workflow call - Downloads results from code coverage analysis and linting, @@ -92,11 +105,87 @@ Workflows - Workflow call - Runs long-running checks, which typically involve an Exasol database instance. +Workflow Extensions +^^^^^^^^^^^^^^^^^^^ + +To use a workflow extension, a user must simply add the file to their project's +``.github/workflows`` directory. The PTB checks that this file exists, and if it does, +then it automatically activates calling that workflow in the relevant parent workflow. +These files are not otherwise checked or maintained by the PTB. The purpose behind the +extensions is to allow project customization in a way that delineates what comes from +and is maintained by the PTB and what is project-specific. + +.. list-table:: + :widths: 25 25 50 + :header-rows: 1 + + * - Filename + - Run on + - Description + * - ``fast-tests-extension.yml`` + - Workflow call + - This extends the ``fast-tests.yml`` and should include additional fast tests. + * - ``merge-gate-extension.yml`` + - Workflow call + - This extends the ``merge-gate.yml`` and the ``needs`` criteria of the job + ``allow-merge``. This extension is used to define additional requirements + to the ``merge-gate``, and it is most likely to include costly slows checks or + tests. It's encouraged to add to this workflow extension additional approval + requests, similar to ``approve-run-slow-tests``. + .. _ci_actions: CI Actions ---------- +.. _pr_merge_yml: + +Merge +^^^^^ + +When a pull request is merged to main, then the ``pr-merge.yml`` workflow is activated. + +.. mermaid:: + :name: merge-diagram + + graph TD + %% Workflow Triggers (Solid Lines) + pr-merge[pr-merge.yml] --> publish-docs[publish-docs.yml] + +.. _periodic_validation_yml: + +Periodic Validation +^^^^^^^^^^^^^^^^^^^ + +Once a week, this `periodic-validation.yml` is triggered on the default branch. Its main +purpose is to ensure that our critical checks and tests continue to run, but it also +sends the results of the linting tools and test coverage to Sonar for an overall report. + +.. literalinclude:: ../../../../exasol/toolbox/templates/github/workflows/periodic-validation.yml + :language: yaml + :start-at: schedule: + :end-at: - cron: "0 0 * * 6" + +.. mermaid:: + + graph TD + %% Define Nodes + checks[checks.yml] + periodic_validation[periodic-validation.yml] + fast-tests[fast-tests.yml] + slow_checks[slow-checks.yml] + report[report.yml] + + %% Workflow Triggers + periodic_validation --> checks + periodic_validation --> fast-tests + periodic_validation --> slow_checks + + %% Dependencies + checks -.->|needs| report + fast-tests -.->|needs| report + slow_checks -.->|needs| report + .. _ci_yml: Pull Request @@ -129,23 +218,29 @@ then the subsequent jobs will not be started. graph TD %% Define Nodes + checks[checks.yml] ci_job[ci.yml] + fast-report[1st call to report.yml] + fast-tests[fast-tests.yml] gate[merge-gate.yml] - checks[checks.yml] slow_run[run-slow-tests] slow_checks[slow-checks.yml] - report[report.yml] + report[2nd call to report.yml] + approver[approve-merge] %% Workflow Triggers ci_job --> gate gate --> checks + gate --> fast-tests gate --> slow_run slow_run -.->|needs| slow_checks %% Dependencies - checks -.->|needs| report checks -.->|needs| approver + checks -.->|needs| fast-report + fast-tests -.->|needs| fast-report + fast-tests -.->|needs| approver slow_checks -.->|needs| approver approver -.->|needs| report @@ -153,24 +248,6 @@ then the subsequent jobs will not be started. style approver fill:#fff,stroke:#333,stroke-dasharray: 5 5 style slow_run fill:#fff,stroke:#333,stroke-dasharray: 5 5 -.. _pr_merge_yml: - -Merge -^^^^^ - -When a pull request is merged to main, then the ``pr-merge.yml`` workflow is activated. - -.. mermaid:: - :name: merge-diagram - - graph TD - %% Workflow Triggers (Solid Lines) - pr-merge[pr-merge.yml] --> checks[checks.yml] - pr-merge --> publish-docs[publish-docs.yml] - - %% Dependencies / Waiting (Dotted Lines) - checks -.->|needs| report[report.yml] - .. _cd_yml: Release diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index dd2427ff9..dec28063a 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -279,11 +279,22 @@ def github_template_dict(self) -> dict[str, Any]: Dictionary of variables to dynamically render Jinja2 templates into valid YAML configurations. """ + fast_tests_extension = ( + self.github_workflow_directory / "fast-tests-extension.yml" + ) + merge_gate_extension = ( + self.github_workflow_directory / "merge-gate-extension.yml" + ) + return { "dependency_manager_version": self.dependency_manager.version, "minimum_python_version": self.minimum_python_version, "os_version": self.os_version, "python_versions": self.python_versions, + "workflow_extension": { + "fast_tests": fast_tests_extension.is_file(), + "merge_gate": merge_gate_extension.is_file(), + }, } @computed_field # type: ignore[misc] @@ -296,6 +307,6 @@ def github_workflow_patcher_yaml(self) -> Path | None: Modification includes replacing and inserting steps. """ workflow_patcher_yaml = self.root_path / ".workflow-patcher.yml" - if workflow_patcher_yaml.exists(): + if workflow_patcher_yaml.is_file(): return workflow_patcher_yaml return None diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml index ee09b0e9f..be669a16a 100644 --- a/exasol/toolbox/templates/github/workflows/checks.yml +++ b/exasol/toolbox/templates/github/workflows/checks.yml @@ -187,37 +187,3 @@ jobs: - name: Build Package id: build-package run: poetry run -- nox -s package:check - - run-unit-tests: - name: Unit Tests (Python-${{ matrix.python-versions }}) - runs-on: "(( os_version ))" - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-versions: (( python_versions | tojson )) - - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "(( dependency_manager_version ))" - - - name: Run Unit Tests - id: run-unit-tests - run: poetry run -- nox -s test:unit -- --coverage - - - name: Upload Artifacts - id: upload-artifacts - uses: actions/upload-artifact@v7 - with: - name: coverage-python${{ matrix.python-versions }}-fast - path: .coverage - include-hidden-files: true diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index 8b93f291f..b6171e32b 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -3,26 +3,16 @@ name: CI on: pull_request: types: [opened, synchronize, reopened] - schedule: - # At 00:00 on every 7th day-of-month from 1 through 31. (https://crontab.guru) - - cron: "0 0 1/7 * *" jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - with: - root-event: ${{ github.event_name }} secrets: inherit permissions: contents: read report: - # Job merge-gate requires manual approval for running the slow checks. If - # current workflow ci.yml is triggered by schedule, there is no manual - # interaction, manual approval will never be given, slow checks will not - # be executed, merge-gate will never terminate, and the report will never - # be called. name: Report needs: - merge-gate diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml new file mode 100644 index 000000000..24386e17b --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -0,0 +1,46 @@ +name: Fast-Tests + +on: + workflow_call: + +jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v7 + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "(( dependency_manager_version ))" + + - name: Run Unit Tests + id: run-unit-tests + run: poetry run -- nox -s test:unit -- --coverage + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python${{ matrix.python-versions }}-fast + path: .coverage + include-hidden-files: true + +(% if workflow_extension.fast_tests %) + fast-tests-extension: + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: read + (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index c6f01ed9b..f58a2d54d 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -2,12 +2,6 @@ name: Merge-Gate on: workflow_call: - inputs: - root-event: - description: GitHub event triggering the root workflow ci.yml - required: false - type: string - default: unknown jobs: run-fast-checks: @@ -16,10 +10,17 @@ jobs: permissions: contents: read + run-fast-tests: + name: Fast Tests + uses: ./.github/workflows/fast-tests.yml + permissions: + contents: read + fast-report: name: Fast Report needs: - run-fast-checks + - run-fast-tests uses: ./.github/workflows/report.yml secrets: inherit permissions: @@ -27,7 +28,6 @@ jobs: approve-run-slow-tests: name: Approve Running Slow Tests? - if: ${{ inputs.root-event != 'schedule' }} runs-on: "(( os_version ))" permissions: contents: read @@ -48,6 +48,14 @@ jobs: permissions: contents: read + (% if workflow_extension.merge_gate %) + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + secrets: inherit + permissions: + contents: read + (% endif %) + # This job ensures inputs have been executed successfully. allow-merge: name: Allow Merge @@ -59,6 +67,9 @@ jobs: needs: - run-fast-checks - run-slow-checks + (% if workflow_extension.merge_gate %) + - merge-gate-extension + (% endif %) # To prevent accidentally merges, this step is required. For more details # see: https://github.com/exasol/python-toolbox/issues/563 steps: diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml new file mode 100644 index 000000000..219de1bd6 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -0,0 +1,37 @@ +name: Periodic-Validation + +on: + schedule: + # At 00:00 on Saturday. (https://crontab.guru) + - cron: "0 0 * * 6" + +jobs: + run-fast-checks: + name: Fast Checks + uses: ./.github/workflows/checks.yml + permissions: + contents: read + + run-fast-tests: + name: Fast Tests + uses: ./.github/workflows/fast-tests.yml + permissions: + contents: read + + run-slow-checks: + name: Slow Checks + uses: ./.github/workflows/slow-checks.yml + secrets: inherit + permissions: + contents: read + + report: + name: Report + needs: + - run-fast-checks + - run-fast-tests + - run-slow-checks + uses: ./.github/workflows/report.yml + secrets: inherit + permissions: + contents: read diff --git a/exasol/toolbox/templates/github/workflows/pr-merge.yml b/exasol/toolbox/templates/github/workflows/pr-merge.yml index 482647ee8..861e45e8a 100644 --- a/exasol/toolbox/templates/github/workflows/pr-merge.yml +++ b/exasol/toolbox/templates/github/workflows/pr-merge.yml @@ -7,14 +7,6 @@ on: - 'master' jobs: - # This job can be removed if certain preconditions are met. See - # https://exasol.github.io/python-toolbox/user_guide/workflows.html#pr-merge-workflow - run-fast-checks: - name: Run Fast Checks - uses: ./.github/workflows/checks.yml - permissions: - contents: read - publish-docs: name: Publish Documentation uses: ./.github/workflows/gh-pages.yml @@ -22,11 +14,3 @@ jobs: contents: read pages: write id-token: write - - report: - needs: - - run-fast-checks - uses: ./.github/workflows/report.yml - secrets: inherit - permissions: - contents: read diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index a3b5b7fb1..49a3f2696 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -30,6 +30,10 @@ # dictionary. If not, then a `jinja2.exceptions.UndefinedError` exception # will be raised. undefined=StrictUndefined, + block_start_string="(%", + block_end_string="%)", + trim_blocks=True, # Removes the newline after a block + lstrip_blocks=True, # Removes tabs/spaces before a block ) diff --git a/test/integration/project-template/nox_test.py b/test/integration/project-template/nox_test.py index 9313f28cb..f1ecc253a 100644 --- a/test/integration/project-template/nox_test.py +++ b/test/integration/project-template/nox_test.py @@ -83,4 +83,4 @@ def test_install_github_workflows(self, poetry_path, run_command): assert output.returncode == 0 file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines() - assert len(file_list) == 13 + assert len(file_list) == 15 diff --git a/test/unit/config_test.py b/test/unit/config_test.py index be6a326bf..0b80c5a05 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -49,6 +49,7 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "3.13", "3.14", ), + "workflow_extension": {"fast_tests": False, "merge_gate": False}, }, "minimum_python_version": "3.10", "os_version": "ubuntu-24.04", diff --git a/test/unit/nox/_workflow_test.py b/test/unit/nox/_workflow_test.py index c4a048719..64fa13acf 100644 --- a/test/unit/nox/_workflow_test.py +++ b/test/unit/nox/_workflow_test.py @@ -35,7 +35,7 @@ class TestGenerateWorkflow: @staticmethod @pytest.mark.parametrize( "nox_session_runner_posargs, expected_count", - [(ALL, 13), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], + [(ALL, 15), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], indirect=["nox_session_runner_posargs"], ) def test_works_as_expected( diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index a78bc1813..eed3331ff 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -251,6 +251,123 @@ def test_updates_jinja_variables(test_yml, yaml_renderer): yaml_dict = yaml_renderer.get_yaml_dict() assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod + def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if workflow_extension.fast_tests %) + fast-tests-extension: + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: read + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + """ + + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + + @staticmethod + def test_adds_jinja_block(test_yml, project_config): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if workflow_extension.fast_tests %) + fast-tests-extension: + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: read + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + fast-tests-extension: + uses: ./.github/workflows/fast-tests-extension.yml + permissions: + contents: read + + """ + workflow_directory = project_config.github_workflow_directory + workflow_directory.mkdir(parents=True) + (workflow_directory / "fast-tests-extension.yml").touch() + + content = cleandoc(input_yaml) + test_yml.write_text(content) + yaml_renderer = YamlRenderer( + github_template_dict=project_config.github_template_dict, + file_path=test_yml, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """ diff --git a/test/unit/util/workflows/templates_test.py b/test/unit/util/workflows/templates_test.py index 994777e26..9e74e222e 100644 --- a/test/unit/util/workflows/templates_test.py +++ b/test/unit/util/workflows/templates_test.py @@ -11,11 +11,13 @@ def test_get_workflow_templates(project_config): "check-release-tag", "checks", "ci", + "fast-tests", "gh-pages", "matrix-all", "matrix-exasol", "matrix-python", "merge-gate", + "periodic-validation", "pr-merge", "report", "slow-checks",