analyses-snapshot-test.yaml

name: Analyses Snapshot Test

on:
  workflow_dispatch:
    inputs:
      OPEN_PR_ON_FAILURE:
        description: 'If the test fails, open a PR to update the snapshots'
        type: boolean
        required: true
        default: false
  schedule:
    - cron: '26 7 * * *' # 7:26 AM UTC
  pull_request:
    paths:
      - 'api/**'
      - '!api/tests/**'
      - '!api/docs/**'
      - '!api/release-notes-internal.md'
      - '!api/release-notes.md'
      - 'shared-data/**'
      - '!shared-data/js/**'
      - '.github/workflows/analyses-snapshot-test.yaml'
      - 'analyses-snapshot-testing/**'

    types:
      - opened #default
      - synchronize #default
      - reopened #default
      - labeled

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  collect-chunks:
    timeout-minutes: 2
    runs-on: ubuntu-latest
    outputs:
      matrix_json: ${{ steps.set-matrix.outputs.json }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
      - name: Setup UV
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: '3.12'
          enable-cache: true
          cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
      - name: Setup the project
        working-directory: ./analyses-snapshot-testing
        run: make setup
      - name: Generate Chunks
        working-directory: ./analyses-snapshot-testing
        run: make gen-chunks
      - name: Save Chunks
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: chunks
          path: analyses-snapshot-testing/chunks/
      - name: Set Matrix
        id: set-matrix
        run: echo "json=$(jq -c . analyses-snapshot-testing/chunks/matrix.json)" >> $GITHUB_OUTPUT
  diff-from-target-branch:
    timeout-minutes: 2
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - name: Checkout Repository
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
      - name: Are the analyses snapshots in my PR branch in sync with the target branch?
        if: github.event_name == 'pull_request'
        run: |
          git fetch origin ${{ github.event.pull_request.base.ref }}
          DIFF_OUTPUT=$(git diff HEAD origin/${{ github.event.pull_request.base.ref }} -- analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test)
          if [ -n "$DIFF_OUTPUT" ]; then
            echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "- :x: Analyses snapshots do **NOT** match \`${{ github.event.pull_request.base.ref }}\` snapshots." >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "  - Is this because you have not pulled and merged \`${{ github.event.pull_request.base.ref }}\`?" >> $GITHUB_STEP_SUMMARY
            echo "  - Or is this because you have already updated your snapshots and are all good 😊?" >> $GITHUB_STEP_SUMMARY
          else
            echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo ":white_check_mark: **Analyses snapshots match \`${{ github.event.pull_request.base.ref }}\` snapshots.**" >> $GITHUB_STEP_SUMMARY
          fi

  process-chunk:
    timeout-minutes: 5
    needs: collect-chunks
    strategy:
      matrix:
        chunk: ${{ fromJson(needs.collect-chunks.outputs.matrix_json) }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
      - name: Setup UV
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: '3.12'
          enable-cache: true
          cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
      - name: Setup the project
        working-directory: ./analyses-snapshot-testing
        run: make setup
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: chunks
          path: analyses-snapshot-testing/chunks/
      - name: Generate Analyses from Chunk
        working-directory: ./analyses-snapshot-testing
        run: make analyze-chunk CHUNK=${{ matrix.chunk.chunk_id }}
      - name: Upload Analysis Results
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          # Each chunk uploads its files as one artifact named after the chunk
          name: analysis-results-${{ matrix.chunk.chunk_id }}.zip
          path: analyses-snapshot-testing/analysis_results/*
          if-no-files-found: error

  execute-tests:
    timeout-minutes: 15
    needs: process-chunk
    runs-on: ubuntu-latest
    env:
      # If we're running because of workflow_dispatch, use the user input to decide
      # whether to open a PR on failure. Otherwise, there is no user input,
      # so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr'
      OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }}
      PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'edge'}}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
      - name: Setup UV
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: '3.12'
          enable-cache: true
          cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
      - name: Setup the project
        working-directory: ./analyses-snapshot-testing
        run: make setup
      - name: Download All Analysis Results
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          pattern: analysis-results-*
          path: all_analysis_results
      - name: Consolidate Analysis Results
        run: |
          find all_analysis_results -type f -exec cp {} analyses-snapshot-testing/analysis_results/ \;
      - name: List All Gathered Files (for debug)
        run: ls -lh analyses-snapshot-testing/analysis_results/
      - name: Run Test
        id: run_test
        working-directory: analyses-snapshot-testing
        run: make snapshot-test

      - name: Run Audit
        id: run_audit
        working-directory: analyses-snapshot-testing
        run: make snapshot-audit-test

      - name: Upload Report
        if: '!cancelled()'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: test-report
          path: analyses-snapshot-testing/results/

      - name: Handle Test Failure
        id: handle_failure
        if: always() && steps.run_test.outcome == 'failure' && (env.OPEN_PR_ON_FAILURE == 'true' || github.event_name == 'schedule')
        working-directory: analyses-snapshot-testing
        run: make snapshot-test-update

      - name: Create Snapshot update Request
        id: create_pull_request
        if: always() && steps.handle_failure.outcome == 'success' && env.OPEN_PR_ON_FAILURE == 'true' && github.event_name == 'pull_request'
        uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11
        with:
          commit-message: 'fix(analyses-snapshot-testing): heal analyses snapshots'
          title: 'fix(analyses-snapshot-testing): heal ${{ github.event.pull_request.head.ref }} snapshots'
          body: 'This PR was requested on the PR https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
          branch: 'analyses-snapshot-testing/heal-${{ github.event.pull_request.head.ref }}'
          base: ${{ github.event.pull_request.head.ref }}
          add-paths: |
            analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test/

      - name: Comment on feature PR
        if: always() && steps.create_pull_request.outcome == 'success' && github.event_name == 'pull_request'
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const message = 'A PR has been opened to address analyses snapshot changes. Please review the changes here: https://github.com/${{ github.repository }}/pull/${{ steps.create_pull_request.outputs.pull-request-number }}';
            github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: message
            });

      - name: Create Snapshot update Request on edge overnight failure
        if: always() && steps.handle_failure.outcome == 'success' && github.event_name == 'schedule'
        uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11
        with:
          commit-message: 'fix(analyses-snapshot-testing): heal edge snapshots'
          title: 'fix(analyses-snapshot-testing): heal edge snapshots'
          body: 'The edge overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.'
          branch: 'analyses-snapshot-testing/heal-edge'
          base: edge
          add-paths: |
            analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test/