From 776ceec72cffa283c445c77b9bb8913e590de067 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:28:27 -0400 Subject: [PATCH 1/6] ci_scripts: update_doc.py: add Add a new version ofthe update_doc.py script from wheel_builder, converted using Claude. Compared to the old version, we update documentation in docs/packages/.md instead of the longer docs/source/packages/.yaml. AI-Generated: Uses Claude Code Sonnet 5 Signed-off-by: Trevor Gamblin --- ci_scripts/update_doc.py | 254 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 ci_scripts/update_doc.py diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py new file mode 100644 index 0000000..2bfcc9f --- /dev/null +++ b/ci_scripts/update_doc.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 BayLibre, SAS +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +""" +Extract metadata from a just-built riscv64 wheel, add or update the +corresponding docs/packages/.yaml entry with the new version, and open +a pull request with the change. + +docs/packages/generate_packages_doc.py renders this YAML into the published +Markdown pages, so this script only needs to maintain the YAML source of +truth; it never touches docs/packages/*.md or index.md directly. +""" + +import os +import re +import string +import subprocess +import sys +import zipfile +from email.message import Message +from email.parser import Parser +from pathlib import Path + +import yaml + +REPO = "riseproject-dev/python-wheels" +DOCS_DIR = Path("docs/packages") +PACKAGES_FILE = Path("ci_scripts/packages.txt") +ARTIFACTS_PATH = os.environ.get("ARTIFACTS_PATH", "dist") + + +def find_wheel_file(path): + for file in Path(path).glob("*.whl"): + return file + return None + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def normalize_label(label): + """ + https://packaging.python.org/en/latest/specifications/well-known-project-urls/#label-normalization + """ + chars_to_remove = string.punctuation + string.whitespace + removal_map = str.maketrans("", "", chars_to_remove) + return label.translate(removal_map).lower() + + +def extract_license(message): + license = message.get("License-Expression") + if not license: + license = message.get("License", "Unknown") + return license + + +def extract_source_code_url(message): + # Collect all "Project-URL" lines + project_urls = message.get_all("Project-URL", []) + well_known_labels = ["source", "repository", "sourcecode", "github"] + + for entry in project_urls: + try: + label, url = map(str.strip, entry.split(",", 1)) + if normalize_label(label) in well_known_labels: + return url + except ValueError: + continue # skip malformed lines + + # A lot of projects use homepage as source code url. Done in a second + # loop so a homepage entry appearing before a well-known source label + # doesn't win by accident. + for entry in project_urls: + try: + label, url = map(str.strip, entry.split(",", 1)) + if normalize_label(label) == "homepage": + return url + except ValueError: + continue + + return message.get("Home-page") # deprecated fallback, may be None + + +def extract_metadata_from_whl(whl_path): + """ + Extract metadata according to https://packaging.python.org/en/latest/specifications/core-metadata/ + """ + with zipfile.ZipFile(whl_path, "r") as z: + metadata_file = next(f for f in z.namelist() if f.endswith("METADATA")) + content = z.read(metadata_file).decode() + message: Message = Parser().parsestr(content) + return { + "name": message.get("Name"), + "version": message.get("Version"), + "license": extract_license(message), + "source_code": extract_source_code_url(message), + } + + +def find_patch_dir(slug, version): + """ + Look for a `patches//` directory as described in + docs/development.md, trying both a `v`-prefixed and bare version tag. + """ + for tag in (f"v{version}", version): + candidate = Path("patches") / slug / tag + if candidate.exists(): + return candidate + return None + + +def yaml_line(key, value): + """Render a single `key: value` YAML mapping line, quoted as needed.""" + return yaml.safe_dump( + {key: value}, default_flow_style=False, allow_unicode=True + ).rstrip("\n") + + +def render_new_yaml(slug, source_code, license, version, patch_dir): + """Render a brand-new docs/packages/.yaml for a package's first version.""" + lines = [yaml_line("package-name", slug)] + if source_code: + lines.append(yaml_line("source-code", source_code)) + lines.append(yaml_line("license", license)) + lines.append("versions:") + lines.append(f" - {yaml_line('version', version)}") + if patch_dir is not None: + lines.append(" patched:") + return "\n".join(lines) + "\n" + + +def append_version(content, package_data, version, license, patch_dir): + """ + Append a new version entry to the end of an existing package YAML file's + `versions:` list, preserving the rest of the file byte-for-byte. + + Returns None if this exact version is already documented. + """ + existing_versions = { + str(v.get("version")) for v in (package_data.get("versions") or []) + } + if str(version) in existing_versions: + return None + + top_level_license = package_data.get("license") + lines = [f" - {yaml_line('version', version)}"] + if patch_dir is not None: + lines.append(" patched:") + if license and license != top_level_license: + lines.append(f" {yaml_line('license', license)}") + + return content.rstrip("\n") + "\n" + "\n".join(lines) + "\n" + + +def add_to_packages_file(slug): + lines = PACKAGES_FILE.read_text().splitlines() + header_end = next(i for i, line in enumerate(lines) if line and not line.startswith("#")) + header, entries = lines[:header_end], [line for line in lines[header_end:] if line] + entries = sorted(set(entries) | {slug}, key=str.casefold) + PACKAGES_FILE.write_text("\n".join(header + entries) + "\n") + + +def git_run(*args): + subprocess.run(["git", *args], check=True) + + +def configure_git_identity(): + git_run("config", "user.name", "github-actions[bot]") + git_run("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + + +def extract_pr_url(stdout): + for line in stdout.split("\n"): + line = line.strip() + if "github.com" in line and "/pull/" in line: + return line + return None + + +def main(): + whl_file = find_wheel_file(ARTIFACTS_PATH) + if not whl_file: + print(f"No .whl file found in {ARTIFACTS_PATH}") + sys.exit(1) + + metadata = extract_metadata_from_whl(whl_file) + display_name = metadata["name"] + version = metadata["version"] + license = metadata["license"] + source_code = metadata["source_code"] + + if not display_name or not version: + print("Name or version could not be extracted") + sys.exit(1) + + slug = normalize_name(display_name) + patch_dir = find_patch_dir(slug, version) + yaml_path = DOCS_DIR / f"{slug}.yaml" + is_new = not yaml_path.exists() + + if is_new: + yaml_path.write_text( + render_new_yaml(slug, source_code, license, version, patch_dir) + ) + else: + content = yaml_path.read_text() + package_data = yaml.safe_load(content) or {} + updated = append_version(content, package_data, version, license, patch_dir) + if updated is None: + print(f"{slug} {version} is already documented; nothing to do") + return + yaml_path.write_text(updated) + + configure_git_identity() + + branch = f"github-actions/{'add' if is_new else 'update'}-doc-for-{slug}" + git_run("switch", "-c", branch) + git_run("add", str(yaml_path)) + + if is_new: + add_to_packages_file(slug) + git_run("add", str(PACKAGES_FILE)) + git_run("commit", "-s", "-m", f"docs: add {slug}\n\nAdd version {version}") + else: + git_run("commit", "-s", "-m", f"docs: update {slug}\n\nAdd version {version}") + + git_run("push", "origin", branch) + + result = subprocess.run( + [ + "gh", "pr", "create", "--draft", + "--repo", REPO, + "--base", "main", + "--head", branch, + "--reviewer", "threexc,justeph", + "--title", f"docs: {'add' if is_new else 'update'} {slug}", + "--body", + "Automatically generated PR to document a newly published wheel. " + "Please review it carefully before merging.\n\n" + "If necessary, force-push this branch.", + ], + capture_output=True, text=True, check=True, + ) + pr_url = extract_pr_url(result.stdout) + print(f"[+] Opened PR: {pr_url or '(URL not found in output)'}") + + +if __name__ == "__main__": + main() From 3fe2a70f31503bd05646a8da5f1b68265fa23fb5 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:37:20 -0400 Subject: [PATCH 2/6] build-numpy.yml: add doc update step Signed-off-by: Trevor Gamblin --- .github/workflows/build-numpy.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index 974a15f..d590d41 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -103,9 +103,15 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: - contents: read + contents: write + pull-requests: write steps: + - name: Checkout python-wheels + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download wheels uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -121,3 +127,8 @@ jobs: gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} files: | dist/*.whl + + - name: Open docs update PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 ci_scripts/update_doc.py From d1ed0551e97048a7ee01082bb1ae102cbb9ebbec Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:40:54 -0400 Subject: [PATCH 3/6] actions: publish-wheels: add composite publishing action Signed-off-by: Trevor Gamblin --- actions/publish-wheels/action.yml | 99 +++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 actions/publish-wheels/action.yml diff --git a/actions/publish-wheels/action.yml b/actions/publish-wheels/action.yml new file mode 100644 index 0000000..8dfcde3 --- /dev/null +++ b/actions/publish-wheels/action.yml @@ -0,0 +1,99 @@ +name: 'Publish riscv64 Wheels and Document Release' +description: > + Checks out python-wheels, downloads a package's built wheel artifacts, + publishes them to the GitLab PyPI Package Registry, and opens a pull + request documenting the new version in docs/packages/. Wraps the publish + job body shared by every build-.yml workflow so it doesn't need + to be duplicated per package. + +inputs: + + # ── Required ──────────────────────────────────────────────────────────────── + + artifact-pattern: + description: > + Pattern passed to actions/download-artifact to select this package's + wheel artifacts, e.g. "numpy-2.5.1-*-manylinux_riscv64". + required: true + + gitlab-username: + description: Passed through to the publish-to-gitlab action. + required: true + + gitlab-token: + description: Passed through to the publish-to-gitlab action. + required: true + + gitlab-project-id: + description: Passed through to the publish-to-gitlab action. + required: true + + gh-token: + description: > + GitHub token used to push the docs branch and open the docs PR. + Composite actions cannot read the `secrets` context directly, so the + caller must pass it explicitly (e.g. secrets.GITHUB_TOKEN). + required: true + + # ── Optional ──────────────────────────────────────────────────────────────── + + artifact-path: + description: Directory the wheel artifacts are downloaded into. + required: false + default: 'dist' + + files: + description: Newline-separated glob(s) of files to publish. Passed through to publish-to-gitlab. + required: false + default: 'dist/*.whl' + + skip-existing: + description: Passed through to the publish-to-gitlab action. + required: false + default: 'false' + + twine-version: + description: Passed through to the publish-to-gitlab action. + required: false + default: '' + +runs: + using: 'composite' + steps: + + - name: Checkout python-wheels + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ inputs.artifact-pattern }} + path: ${{ inputs.artifact-path }} + merge-multiple: true + + - name: Publish to GitLab PyPI registry + uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + with: + gitlab-username: ${{ inputs.gitlab-username }} + gitlab-token: ${{ inputs.gitlab-token }} + gitlab-project-id: ${{ inputs.gitlab-project-id }} + files: ${{ inputs.files }} + skip-existing: ${{ inputs.skip-existing }} + twine-version: ${{ inputs.twine-version }} + + - uses: actions/setup-python@v5 + with: + python-version: '3' + + - name: Install ci_scripts/update_doc.py dependencies + shell: bash + run: pip install pyyaml + + - name: Open docs update PR + shell: bash + env: + GH_TOKEN: ${{ inputs.gh-token }} + ARTIFACTS_PATH: ${{ inputs.artifact-path }} + run: python3 ci_scripts/update_doc.py From c4399b1e9d06b1059d1278e990fee198226bbd0a Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:41:25 -0400 Subject: [PATCH 4/6] workflows: build-numpy.yml: use publish-wheels Signed-off-by: Trevor Gamblin --- .github/workflows/build-numpy.yml | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index d590d41..d6960bc 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -12,6 +12,8 @@ on: paths: - '.github/workflows/build-numpy.yml' - 'actions/publish-to-gitlab/**' + - 'actions/publish-wheels/**' + - 'ci_scripts/update_doc.py' concurrency: group: ${{ github.workflow }}-${{ inputs.version || '2.5.1' }}-${{ github.head_ref || github.run_id }} @@ -107,28 +109,11 @@ jobs: pull-requests: write steps: - - name: Checkout python-wheels - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Download wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 - path: dist - merge-multiple: true - - - name: Publish to GitLab PyPI registry - uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main with: + artifact-pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} - files: | - dist/*.whl - - - name: Open docs update PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python3 ci_scripts/update_doc.py + gh-token: ${{ secrets.GITHUB_TOKEN }} From a7233115a2657c6bf4062b1f2e3f5c800e513f3c Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:46:34 -0400 Subject: [PATCH 5/6] docs: development.md: update publish example Signed-off-by: Trevor Gamblin --- docs/development.md | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/development.md b/docs/development.md index 0c76159..bfb4b84 100644 --- a/docs/development.md +++ b/docs/development.md @@ -129,9 +129,16 @@ when invoked. ### Using the python-wheels Repository in Workflows The `python-wheels` repository contains some custom Actions we require, and -patch files to apply for certain projects. The most critical example is the -`publish-to-gitlab` Action. With it in place, the `build-numpy.yml` script's -`publish` job looks like this: +patch files to apply for certain projects. The one every `build-.yml` +workflow needs is `publish-wheels`, which performs the following steps: + +1. Downloads the built wheel(s) from the previous job +2. Uploads them to the GitLab PyPI registry (via the lower-level + `publish-to-gitlab` Action) +3. Opens a PR against `docs/packages/.yaml` documenting the new version + (via `ci_scripts/update_doc.py`), which `docs/packages/generate_packages_doc.py` + later renders into the published Markdown page. With it in place, the + `build-numpy.yml` script's `publish` job looks like this: ``` publish: @@ -143,29 +150,28 @@ publish: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: - contents: read + contents: write + pull-requests: write steps: - - name: Download wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 - path: dist - merge-multiple: true - - - name: Publish to GitLab PyPI registry - uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main with: + artifact-pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} - files: | - dist/*.whl + gh-token: ${{ secrets.GITHUB_TOKEN }} ``` -Other workflows need to follow a similar process - checkout the `python-wheels` -repo, and run the `publish-to-gitlab` action to upload built wheels to the RISE -Python registry. +`permissions` needs `contents: write` and `pull-requests: write` here (not just +`contents: read`) since the docs step pushes a branch and opens a PR with the +default `GITHUB_TOKEN`. + +Other workflows need to follow the same process, modifying `artifact-pattern` to +match their own artifact naming scheme and otherwise reusing `publish-wheels` +like the example. The `publish-to-gitlab` Action should only be used directly if +a workflow needs the upload step without the docs PR side effect. ## Testing a New Workflow From f11628f1faea5ebdd5c528e213b4ef9150397594 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Thu, 16 Jul 2026 14:06:12 -0400 Subject: [PATCH 6/6] workflows: build-numpy: trigger only on changes to workflow Don't trigger the build-numpy.yml workflow if a change has been made to other files, such as doc update scripts or documentation itself. Signed-off-by: Trevor Gamblin --- .github/workflows/build-numpy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index d6960bc..3d7318c 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -11,9 +11,6 @@ on: pull_request: paths: - '.github/workflows/build-numpy.yml' - - 'actions/publish-to-gitlab/**' - - 'actions/publish-wheels/**' - - 'ci_scripts/update_doc.py' concurrency: group: ${{ github.workflow }}-${{ inputs.version || '2.5.1' }}-${{ github.head_ref || github.run_id }}