diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c92fb6b05838..37a031bd042d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,8 +28,6 @@ concurrency: env: HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} - # WARNING: this disables cross os caching as ~ and - # github.workspace evaluate to differents paths CACHED_DEPENDENCY_PATHS: | ${{ github.workspace }}/node_modules ${{ github.workspace }}/packages/*/node_modules @@ -38,25 +36,23 @@ env: # DEPENDENCY_CACHE_KEY: can't be set here because we don't have access to yarn.lock - # WARNING: this disables cross os caching as ~ and - # github.workspace evaluate to differents paths - # packages/utils/cjs and packages/utils/esm: Symlinks to the folders inside of `build`, needed for tests - CACHED_BUILD_PATHS: | - ${{ github.workspace }}/dev-packages/*/build - ${{ github.workspace }}/packages/*/build - ${{ github.workspace }}/packages/*/lib - ${{ github.workspace }}/packages/ember/*.d.ts - ${{ github.workspace }}/packages/gatsby/*.d.ts - - BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }} - - # GH will use the first restore-key it finds that matches - # So it will start by looking for one from the same branch, else take the newest one it can find elsewhere - # We want to prefer the cache from the current develop branch, if we don't find any on the current branch - NX_CACHE_RESTORE_KEYS: | - nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }} - nx-Linux-${{ github.ref }} - nx-Linux + # build-output artifact paths are computed in job_build (step output from + # yarn ci:print-build-artifact-paths — Nx merged outputs for build:transpile, and build:types). + + # upload-artifact globs drop the path through the first `*` (see upload-artifact + # README). Tarballs therefore land in the artifact as /*.tgz; download + # build-tarball-output into packages/ so they resolve to packages//*.tgz. + TARBALL_ARTIFACT_GLOB: packages/*/*.tgz + TARBALL_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages + + BUILD_LAYER_PATH: ${{ github.workspace }}/packages/aws-serverless/build/aws/dist-serverless + + # Same glob / download split as TARBALL_ARTIFACT_*: upload-artifact strips the path through + # the first `*`, so bundle trees are stored as /build/bundles/...; download into packages/. + BUNDLE_ARTIFACT_GLOB: packages/*/build/bundles + BUNDLE_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages + + NX_CACHE_KEY: nx-Linux-${{ github.head_ref || github.ref }}-${{ github.run_id }} # https://bsky.app/profile/joyeecheung.bsky.social/post/3lhy6o54fo22h # Apparently some of our CI failures are attributable to a corrupt v8 cache, causing v8 failures with: "Check failed: current == end_slot_index.". @@ -66,6 +62,7 @@ env: jobs: job_get_metadata: uses: ./.github/workflows/ci-metadata.yml + name: Get CI Metadata with: head_commit: ${{ github.event.inputs.commit || github.sha }} permissions: @@ -110,34 +107,51 @@ jobs: base: ${{ github.event.pull_request.base.sha }} head: ${{ env.HEAD_COMMIT }} - - name: NX cache - uses: actions/cache@v5 - # Disable cache when: - # - on release branches - # - when PR has `ci-skip-cache` label or on nightly builds - if: | - needs.job_get_metadata.outputs.is_release == 'false' && - needs.job_get_metadata.outputs.force_skip_cache == 'false' - with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT || github.sha }} - # On develop branch, we want to _store_ the cache (so it can be used by other branches), but never _restore_ from it - restore-keys: - ${{needs.job_get_metadata.outputs.is_base_branch == 'false' && env.NX_CACHE_RESTORE_KEYS || - 'nx-never-restore'}} - - name: Build packages - run: yarn build + run: yarn build:ci + + - name: Compute build artifact paths from Nx + id: nx_build_paths + run: | + { + echo 'paths<> "$GITHUB_OUTPUT" + + - name: Store NX cache + uses: actions/cache/save@v5 + # Only cache this per-PR to speed up CI. + if: github.event_name == 'pull_request' + with: + path: | + .nx/cache + .nx/workspace-data + key: ${{ env.NX_CACHE_KEY }} - name: Upload build artifacts uses: actions/upload-artifact@v7 with: name: build-output - path: ${{ env.CACHED_BUILD_PATHS }} + path: ${{ steps.nx_build_paths.outputs.paths }} retention-days: 4 compression-level: 6 overwrite: true + - name: Determine which test applications should be run + id: matrix + run: + yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && + github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT + working-directory: dev-packages/e2e-tests + + - name: Determine which optional E2E test applications should be run + id: matrix-optional + run: + yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && + github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT + working-directory: dev-packages/e2e-tests + outputs: dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }} changed_node_integration: @@ -164,6 +178,92 @@ jobs: changed_browser_integration: ${{ needs.job_get_metadata.outputs.changed_ci == 'true' || contains(steps.checkForAffected.outputs.affected, '@sentry-internal/browser-integration-tests') }} + changed_aws_serverless: + ${{ needs.job_get_metadata.outputs.changed_ci == 'true' || contains(steps.checkForAffected.outputs.affected, + '@sentry/aws-serverless') }} + e2e-matrix: ${{ steps.matrix.outputs.matrix }} + e2e-matrix-optional: ${{ steps.matrix-optional.outputs.matrix }} + + job_build_layer: + name: Build Lambda layer + needs: [job_get_metadata, job_build] + if: + needs.job_build.outputs.changed_aws_serverless == 'true' || contains(needs.job_build.outputs.e2e-matrix, + 'aws-serverless') || github.event_name != 'pull_request' + timeout-minutes: 10 + runs-on: ubuntu-24.04 + steps: + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) + uses: actions/checkout@v6 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'package.json' + - name: Restore NX cache + uses: actions/cache/restore@v5 + with: + path: | + .nx/cache + .nx/workspace-data + key: ${{ env.NX_CACHE_KEY }} + + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + + - name: Build Lambda layer + run: yarn build:layer + + - name: Upload build artifacts + uses: actions/upload-artifact@v7 + with: + name: build-layer-output + path: ${{ env.BUILD_LAYER_PATH }} + retention-days: 4 + compression-level: 6 + overwrite: true + + job_build_bundles: + name: Build bundles + needs: [job_get_metadata, job_build] + timeout-minutes: 10 + runs-on: ubuntu-24.04 + steps: + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) + uses: actions/checkout@v6 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'package.json' + - name: Restore NX cache + uses: actions/cache/restore@v5 + with: + path: | + .nx/cache + .nx/workspace-data + key: ${{ env.NX_CACHE_KEY }} + + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + + - name: Build bundles + run: yarn build:bundle + + - name: Upload build artifacts + uses: actions/upload-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_GLOB }} + retention-days: 4 + compression-level: 6 + overwrite: true job_check_branches: name: Check PR branches @@ -182,7 +282,7 @@ jobs: job_size_check: name: Size Check - needs: [job_get_metadata, job_build] + needs: [job_get_metadata, job_build, job_build_bundles] timeout-minutes: 15 runs-on: ubuntu-24.04 if: @@ -201,6 +301,11 @@ jobs: uses: ./.github/actions/restore-cache with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + - name: Restore build bundle artifacts + uses: actions/download-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_DOWNLOAD_PATH }} - name: Check bundle sizes uses: ./dev-packages/size-limit-gh-action with: @@ -298,7 +403,7 @@ jobs: job_artifacts: name: Upload Artifacts - needs: [job_get_metadata, job_build] + needs: [job_get_metadata, job_build, job_build_layer, job_build_bundles, job_build_tarballs] runs-on: ubuntu-24.04 # Build artifacts are only needed for releasing workflow. if: needs.job_get_metadata.outputs.is_release == 'true' @@ -316,8 +421,23 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Pack tarballs - run: yarn build:tarball + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 + with: + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} + + - name: Restore build bundle artifacts + uses: actions/download-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_DOWNLOAD_PATH }} + + - name: Restore build layer artifacts + uses: actions/download-artifact@v7 + with: + name: build-layer-output + path: ${{ env.BUILD_LAYER_PATH }} - name: Archive artifacts uses: actions/upload-artifact@v7 @@ -478,7 +598,7 @@ jobs: name: Playwright ${{ matrix.bundle }}${{ matrix.project && matrix.project != 'chromium' && format(' {0}', matrix.project) || ''}}${{ matrix.shard && format(' ({0}/{1})', matrix.shard, matrix.shards) || ''}} Tests - needs: [job_get_metadata, job_build] + needs: [job_get_metadata, job_build, job_build_bundles] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04-large-js timeout-minutes: 25 @@ -543,6 +663,12 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + - name: Restore build bundle artifacts + uses: actions/download-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_DOWNLOAD_PATH }} + - name: Install Playwright uses: ./.github/actions/install-playwright with: @@ -581,7 +707,7 @@ jobs: job_browser_loader_tests: name: PW ${{ matrix.bundle }} Tests - needs: [job_get_metadata, job_build] + needs: [job_get_metadata, job_build, job_build_bundles] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -611,6 +737,12 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + - name: Restore build bundle artifacts + uses: actions/download-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_DOWNLOAD_PATH }} + - name: Install Playwright uses: ./.github/actions/install-playwright with: @@ -662,7 +794,7 @@ jobs: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - name: Check for dts files that reference stuff in the temporary build folder run: | - if grep -r --include "*.d.ts" --exclude-dir ".nxcache" 'import("@sentry(-internal)?/[^/]*/build' .; then + if grep -r --include "*.d.ts" --exclude-dir ".nx" 'import("@sentry(-internal)?/[^/]*/build' .; then echo "Found illegal TypeScript import statement." exit 1 fi @@ -834,25 +966,17 @@ jobs: cd packages/remix yarn test:integration:ci - job_e2e_prepare: - name: Prepare E2E tests + job_build_tarballs: + name: Build tarballs # We want to run this if: # - The build job was successful, not skipped if: | always() && needs.job_build.result == 'success' needs: [job_get_metadata, job_build] - runs-on: ubuntu-24.04-large-js + runs-on: ubuntu-24.04 timeout-minutes: 15 - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - matrix-optional: ${{ steps.matrix-optional.outputs.matrix }} steps: - - name: Check out base commit (${{ github.event.pull_request.base.sha }}) - uses: actions/checkout@v6 - if: github.event_name == 'pull_request' - with: - ref: ${{ github.event.pull_request.base.sha }} - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v6 with: @@ -861,48 +985,39 @@ jobs: uses: actions/setup-node@v6 with: node-version-file: 'package.json' + - name: Restore NX cache + uses: actions/cache/restore@v5 + with: + path: | + .nx/cache + .nx/workspace-data + key: ${{ env.NX_CACHE_KEY }} + - name: Restore caches uses: ./.github/actions/restore-cache with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: NX cache - uses: actions/cache/restore@v5 - with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }} - # On develop branch, we want to _store_ the cache (so it can be used by other branches), but never _restore_ from it - restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }} - name: Build tarballs run: yarn build:tarball - - name: Stores tarballs in cache - uses: actions/cache/save@v5 + - name: Upload tarball artifacts + uses: actions/upload-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} - - - name: Determine which E2E test applications should be run - id: matrix - run: - yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && - github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT - working-directory: dev-packages/e2e-tests - - - name: Determine which optional E2E test applications should be run - id: matrix-optional - run: - yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && - github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT - working-directory: dev-packages/e2e-tests + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_GLOB }} + if-no-files-found: error + retention-days: 4 + compression-level: 6 + overwrite: true job_e2e_tests: name: E2E ${{ matrix.label || matrix.test-application }} Test # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_e2e_prepare.result == 'success' && needs.job_e2e_prepare.outputs.matrix != '{"include":[]}' - needs: [job_get_metadata, job_build, job_e2e_prepare] + always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' + needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -916,7 +1031,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' strategy: fail-fast: false - matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }} + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix) }} steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v6 @@ -950,16 +1065,18 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Restore tarball cache - uses: actions/cache/restore@v5 - id: restore-tarball-cache + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - - name: Build tarballs if not cached - if: steps.restore-tarball-cache.outputs.cache-hit != 'true' - run: yarn build:tarball + - name: Restore build layer artifacts + uses: actions/download-artifact@v7 + if: matrix.test-application == 'aws-serverless' || matrix.test-application == 'aws-serverless-layer' + with: + name: build-layer-output + path: ${{ env.BUILD_LAYER_PATH }} - name: Prepare e2e tests run: yarn test:prepare @@ -1030,10 +1147,10 @@ jobs: # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_e2e_prepare.result == 'success' && - needs.job_e2e_prepare.outputs.matrix-optional != '{"include":[]}' && (github.event_name != 'pull_request' || + always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_build_tarballs.result == 'success' && + needs.job_build.outputs.e2e-matrix-optional != '{"include":[]}' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' - needs: [job_get_metadata, job_build, job_e2e_prepare] + needs: [job_get_metadata, job_build, job_build_tarballs] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -1047,7 +1164,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' strategy: fail-fast: false - matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }} + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }} steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) @@ -1066,16 +1183,11 @@ jobs: with: dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Restore tarball cache - uses: actions/cache/restore@v5 - id: restore-tarball-cache + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 with: - path: ${{ github.workspace }}/packages/*/*.tgz - key: ${{ env.BUILD_CACHE_TARBALL_KEY }} - - - name: Build tarballs if not cached - if: steps.restore-tarball-cache.outputs.cache-hit != 'true' - run: yarn build:tarball + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - name: Prepare E2E tests run: yarn test:prepare @@ -1131,6 +1243,9 @@ jobs: needs: [ job_build, + job_build_bundles, + job_build_layer, + job_build_tarballs, job_browser_unit_tests, job_bun_unit_tests, job_deno_unit_tests, diff --git a/.github/workflows/flaky-test-detector.yml b/.github/workflows/flaky-test-detector.yml index c0a8f1f720b1..d74a82a667e5 100644 --- a/.github/workflows/flaky-test-detector.yml +++ b/.github/workflows/flaky-test-detector.yml @@ -10,11 +10,6 @@ on: env: HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} - NX_CACHE_RESTORE_KEYS: | - nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }} - nx-Linux-${{ github.ref }} - nx-Linux - # Cancel in progress workflows on pull_requests. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value concurrency: @@ -36,16 +31,10 @@ jobs: with: node-version-file: 'package.json' cache: 'yarn' + - name: Install dependencies run: yarn install --ignore-engines --frozen-lockfile - - name: NX cache - uses: actions/cache/restore@v5 - with: - path: .nxcache - key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }} - restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }} - - name: Build packages run: yarn build diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 15de0fd49ee0..2ff4b0665cd3 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -198,7 +198,7 @@ try { ``` Test apps in the folder `test-applications` will be automatically picked up by CI in the job `job_e2e_tests` (in `.github/workflows/build.yml`). -The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.ts`. +The test matrix for CI is generated in `dev-packages/e2e-tests/lib/getTestMatrix.mjs`. For each test app, CI checks its dependencies (and devDependencies) to see if any of them have changed in the current PR (based on nx affected projects). For example, if something is changed in the browser package, only E2E test apps that depend on browser will run, while others will be skipped. diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.ts b/dev-packages/e2e-tests/lib/getTestMatrix.mjs similarity index 73% rename from dev-packages/e2e-tests/lib/getTestMatrix.ts rename to dev-packages/e2e-tests/lib/getTestMatrix.mjs index 86a4bda3e701..b8be7af5f528 100644 --- a/dev-packages/e2e-tests/lib/getTestMatrix.ts +++ b/dev-packages/e2e-tests/lib/getTestMatrix.mjs @@ -1,42 +1,21 @@ -import { execSync } from 'child_process'; -import * as fs from 'fs'; -import { sync as globSync } from 'glob'; -import * as path from 'path'; -import { dirname } from 'path'; -import { parseArgs } from 'util'; - -interface MatrixInclude { - /** The test application (directory) name. */ - 'test-application': string; - /** Optional override for the build command to run. */ - 'build-command'?: string; - /** Optional override for the assert command to run. */ - 'assert-command'?: string; - /** Optional label for the test run. If not set, defaults to value of `test-application`. */ - label?: string; -} +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; -interface PackageJsonSentryTestConfig { - /** If this is true, the test app is optional. */ - optional?: boolean; - /** Variant configs that should be run in non-optional test runs. */ - variants?: Partial[]; - /** Variant configs that should be run in optional test runs. */ - optionalVariants?: Partial[]; - /** Skip this test app for matrix generation. */ - skip?: boolean; -} +const __dirname = path.dirname(fileURLToPath(import.meta.url)); /** - * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests. - * It checks which test applications are affected by the current changes in the PR and then generates a matrix + * Generates a matrix for the GitHub Actions workflow to run the E2E tests. + * Checks which test applications are affected by the current changes in the PR and then generates a matrix * including all test apps that have at least one dependency that was changed in the PR. * If no `--base=xxx` is provided, it will output all test applications. * * If `--optional=true` is set, it will generate a matrix of optional test applications only. * Otherwise, these will be skipped. */ -function run(): void { +function run() { const { values } = parseArgs({ args: process.argv.slice(2), options: { @@ -52,9 +31,7 @@ function run(): void { // eslint-disable-next-line no-console console.error(`Parsed command line arguments: base=${base}, head=${head}, optional=${optional}`); - const testApplications = globSync('*/package.json', { - cwd: `${__dirname}/../test-applications`, - }).map(filePath => dirname(filePath)); + const testApplications = discoverTestApplicationDirs(); // For GitHub Action debugging (using stderr the 'matrix=...' output is not polluted) // eslint-disable-next-line no-console @@ -67,7 +44,7 @@ function run(): void { : testApplications; const optionalMode = optional === 'true'; - const includes: MatrixInclude[] = []; + const includes = []; includedTestApplications.forEach(testApp => { addIncludesForTestApp(testApp, includes, { optionalMode }); @@ -78,11 +55,18 @@ function run(): void { console.log(`matrix=${JSON.stringify({ include: includes })}`); } -function addIncludesForTestApp( - testApp: string, - includes: MatrixInclude[], - { optionalMode }: { optionalMode: boolean }, -): void { +/** Direct children of `test-applications/` that contain a `package.json` (replaces glob one-segment + package.json). */ +function discoverTestApplicationDirs() { + const appsRoot = path.join(__dirname, '..', 'test-applications'); + return fs + .readdirSync(appsRoot, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .filter(name => fs.existsSync(path.join(appsRoot, name, 'package.json'))) + .sort(); +} + +function addIncludesForTestApp(testApp, includes, { optionalMode }) { const packageJson = getPackageJson(testApp); const shouldSkip = packageJson.sentryTest?.skip || false; @@ -108,7 +92,7 @@ function addIncludesForTestApp( }); } -function getSentryDependencies(appName: string): string[] { +function getSentryDependencies(appName) { const packageJson = getPackageJson(appName); const dependencies = { @@ -119,11 +103,7 @@ function getSentryDependencies(appName: string): string[] { return Object.keys(dependencies).filter(key => key.startsWith('@sentry')); } -function getPackageJson(appName: string): { - dependencies?: { [key: string]: string }; - devDependencies?: { [key: string]: string }; - sentryTest?: PackageJsonSentryTestConfig; -} { +function getPackageJson(appName) { const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json'); if (!fs.existsSync(fullPath)) { @@ -133,19 +113,14 @@ function getPackageJson(appName: string): { return JSON.parse(fs.readFileSync(fullPath, 'utf8')); } -run(); - -function getAffectedTestApplications( - testApplications: string[], - { base = 'develop', head }: { base?: string; head?: string }, -): string[] { +function getAffectedTestApplications(testApplications, { base = 'develop', head }) { const additionalArgs = [`--base=${base}`]; if (head) { additionalArgs.push(`--head=${head}`); } - let affectedProjects: string[] = []; + let affectedProjects = []; try { affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`) .toString() @@ -201,7 +176,7 @@ function getAffectedTestApplications( return Array.from(testAppsToRun); } -function getChangedTestApps(base: string, head?: string): false | Set { +function getChangedTestApps(base, head) { const changedFiles = execSync(`git diff --name-only ${base}${head ? `..${head}` : ''} -- .`, { encoding: 'utf-8', }) @@ -214,7 +189,7 @@ function getChangedTestApps(base: string, head?: string): false | Set { // eslint-disable-next-line no-console console.error(`Changed files since ${base}${head ? `..${head}` : ''}: ${JSON.stringify(changedFiles)}`); - const changedTestApps: Set = new Set(); + const changedTestApps = new Set(); const testAppsPrefix = 'dev-packages/e2e-tests/test-applications/'; for (const file of changedFiles) { @@ -233,3 +208,5 @@ function getChangedTestApps(base: string, head?: string): false | Set { return changedTestApps; } + +run(); diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json index 25d87ad10a6e..11097e64ad9b 100644 --- a/dev-packages/e2e-tests/package.json +++ b/dev-packages/e2e-tests/package.json @@ -12,8 +12,8 @@ "test:prepare": "ts-node prepare.ts", "test:validate": "ts-node validate-packed-tarball-setup.ts", "clean": "rimraf tmp node_modules packed && yarn clean:test-applications && yarn clean:pnpm", - "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts", - "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true", + "ci:build-matrix": "node ./lib/getTestMatrix.mjs", + "ci:build-matrix-optional": "node ./lib/getTestMatrix.mjs --optional=true", "ci:copy-to-temp": "ts-node ./ciCopyToTemp.ts", "ci:pnpm-overrides": "ts-node ./ciPnpmOverrides.ts", "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.nuxt,.sveltekit,.react-router,.astro,.output,pnpm-lock.yaml,.last-run.json,test-results,.angular,event-dumps}", diff --git a/nx.json b/nx.json index 7cd807e089fb..08faf7cf2e62 100644 --- a/nx.json +++ b/nx.json @@ -61,7 +61,7 @@ } }, "$schema": "./node_modules/nx/schemas/nx-schema.json", - "cacheDirectory": ".nxcache", + "cacheDirectory": ".nx/cache", "tui": { "autoExit": true }, diff --git a/package.json b/package.json index f455f42e3bbf..6e8c7fcf272c 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,10 @@ { "private": true, "scripts": { - "build": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types build:bundle build:layer", + "build": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types build:bundle", + "build:ci": "node ./scripts/verify-packages-versions.js && nx run-many -t build:transpile build:types", "build:bundle": "nx run-many -t build:bundle", + "build:layer": "nx run-many -t build:layer", "build:dev": "nx run-many -t build:types build:transpile", "build:dev:filter": "nx run-many -t build:dev -p", "build:transpile": "nx run-many -t build:transpile", @@ -11,6 +13,7 @@ "build:dev:watch": "nx run-many -t build:dev:watch", "build:tarball": "run-s clean:tarballs build:tarballs", "build:tarballs": "nx run-many -t build:tarball", + "ci:print-build-artifact-paths": "node ./scripts/ci-print-build-artifact-paths.mjs", "changelog": "ts-node ./scripts/get-commit-list.ts", "generate-changelog": "ts-node ./scripts/generate-changelog.ts", "circularDepCheck": "nx run-many -t circularDepCheck", diff --git a/packages/angular/package.json b/packages/angular/package.json index 6e59e3588fdd..7169e8cf67ed 100644 --- a/packages/angular/package.json +++ b/packages/angular/package.json @@ -71,10 +71,7 @@ "^build:types" ], "outputs": [ - "{projectRoot}/build/esm2015", - "{projectRoot}/build/fesm2015", - "{projectRoot}/build/fesm2020", - "{projectRoot}/build/*.d.ts" + "{projectRoot}/build" ] } } diff --git a/packages/aws-serverless/package.json b/packages/aws-serverless/package.json index db70b13d3e9e..e002a642e6c8 100644 --- a/packages/aws-serverless/package.json +++ b/packages/aws-serverless/package.json @@ -80,11 +80,12 @@ "@vercel/nft": "^1.3.0" }, "scripts": { - "build": "run-p build:transpile build:types build:extension && run-s build:layer", + "build": "run-p build:transpile build:types", "build:extension": "rollup -c rollup.lambda-extension.config.mjs && yarn ts-node scripts/buildLambdaExtension.ts", "build:layer": "rimraf build/aws && yarn ts-node scripts/buildLambdaLayer.ts", "build:dev": "run-p build:transpile build:types", - "build:transpile": "rollup -c rollup.npm.config.mjs", + "build:transpile": "run-s build:transpile:npm build:extension", + "build:transpile:npm": "rollup -c rollup.npm.config.mjs", "build:types": "run-s build:types:core build:types:downlevel", "build:types:core": "tsc -p tsconfig.types.json", "build:types:downlevel": "yarn downlevel-dts build/npm/types build/npm/types-ts3.8 --to ts3.8", @@ -117,20 +118,10 @@ ], "outputs": [ "{projectRoot}/build/npm/esm", - "{projectRoot}/build/npm/cjs" - ] - }, - "build:extension": { - "inputs": [ - "production", - "^production" - ], - "dependsOn": [ - "^build:transpile" - ], - "outputs": [ + "{projectRoot}/build/npm/cjs", "{projectRoot}/build/lambda-extension" - ] + ], + "cache": true }, "build:layer": { "inputs": [ @@ -139,7 +130,7 @@ ], "dependsOn": [ "build:transpile", - "build:extension" + "build:types" ], "outputs": [ "{projectRoot}/build/aws" diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index 3d0b6e7ed305..840fdef5d4aa 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -84,5 +84,24 @@ "volta": { "extends": "../../package.json" }, + "nx": { + "targets": { + "build:transpile": { + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/build/esm", + "{projectRoot}/build/cjs", + "{projectRoot}/*.d.ts" + ], + "dependsOn": [ + "^build:transpile" + ], + "cache": true + } + } + }, "sideEffects": false } diff --git a/packages/nestjs/package.json b/packages/nestjs/package.json index 625f8330a65f..024e25cf3cce 100644 --- a/packages/nestjs/package.json +++ b/packages/nestjs/package.json @@ -85,5 +85,24 @@ "volta": { "extends": "../../package.json" }, + "nx": { + "targets": { + "build:types": { + "inputs": [ + "production", + "^production" + ], + "dependsOn": [ + "^build:types" + ], + "outputs": [ + "{projectRoot}/build/types", + "{projectRoot}/build/types-ts3.8", + "{projectRoot}/*.d.ts" + ], + "cache": true + } + } + }, "sideEffects": false } diff --git a/scripts/ci-print-build-artifact-paths.mjs b/scripts/ci-print-build-artifact-paths.mjs new file mode 100644 index 000000000000..690b995d78b2 --- /dev/null +++ b/scripts/ci-print-build-artifact-paths.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +/** + * Prints multiline paths for actions/upload-artifact `path` (often wired via a prior step's + * `GITHUB_OUTPUT` `paths< yarn ci:print-build-artifact-paths + * (defaults to cwd when GITHUB_WORKSPACE is unset) + */ +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const workspaceRoot = path.resolve(__dirname, '..'); +const graphPath = path.join(workspaceRoot, '.nx', 'ci-print-build-artifact-paths-graph.json'); + +const TARGETS = ['build:transpile', 'build:types']; + +fs.mkdirSync(path.dirname(graphPath), { recursive: true }); +execSync(`yarn nx graph --file="${graphPath}"`, { + cwd: workspaceRoot, + stdio: ['ignore', 'pipe', 'inherit'], +}); + +const { graph } = JSON.parse(fs.readFileSync(graphPath, 'utf8')); +try { + fs.unlinkSync(graphPath); +} catch { + // ignore +} + +/** @type {Map>} key = `${kind}\0${suffix}` */ +const groups = new Map(); + +for (const node of Object.values(graph.nodes)) { + const root = node.data?.root; + if (!root || (!root.startsWith('packages/') && !root.startsWith('dev-packages/'))) { + continue; + } + + const [kind, pkg] = root.split('/'); + if (!kind || !pkg) { + continue; + } + + const targets = node.data?.targets || {}; + for (const targetName of TARGETS) { + const outputs = targets[targetName]?.outputs; + if (!Array.isArray(outputs)) { + continue; + } + + for (const output of outputs) { + const rel = output.replace(/\{projectRoot\}/g, root).replace(/\\/g, '/'); + const prefix = `${kind}/${pkg}/`; + if (!rel.startsWith(prefix)) { + throw new Error(`Unexpected Nx output (missing project prefix): ${rel}`); + } + const suffix = rel.slice(prefix.length); + const key = `${kind}\0${suffix}`; + if (!groups.has(key)) { + groups.set(key, new Set()); + } + groups.get(key).add(pkg); + } + } +} + +const ws = (process.env.GITHUB_WORKSPACE || workspaceRoot).replace(/\\/g, '/'); +const lines = new Set(); + +// A glob like packages + star + slash + "build" matches every package's build tree, so we +// never emit that when several projects each declare a top-level {projectRoot}/build output. +function isUnsafeSharedTopLevelBuildSuffix(suffix, pkgCount) { + return pkgCount > 1 && !suffix.includes('/') && !/[?*]/.test(suffix) && suffix === 'build'; +} + +for (const [key, pkgSet] of groups) { + const [kind, suffix] = key.split('\0'); + const pkgs = [...pkgSet].sort((a, b) => a.localeCompare(b)); + const n = pkgs.length; + + if (n === 1) { + lines.add(`${ws}/${kind}/${pkgs[0]}/${suffix}`); + continue; + } + + if (isUnsafeSharedTopLevelBuildSuffix(suffix, n)) { + for (const pkg of pkgs) { + lines.add(`${ws}/${kind}/${pkg}/build`); + } + continue; + } + + lines.add(`${ws}/${kind}/*/${suffix}`); +} + +process.stdout.write([...lines].sort((a, b) => a.localeCompare(b)).join('\n')); +if (lines.size) { + process.stdout.write('\n'); +}