[WIP] Gaia charts

.envrc

@@ -0,0 +1,4 @@
+use nix
+watch_file flake.nix flake.lock shell.nix
+watch_file nixos/pkgs/uv-python.nix
+watch_file python/pyproject.toml python/uv.lock

.github/scripts/update_manifest.py

@@ -4,11 +4,19 @@
 import argparse
 import json
 import re
+import urllib.error
+import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 
 
 STORE_PATH_RE = re.compile(r"^/nix/store/[a-z0-9]+-[A-Za-z0-9._+=?,-]+$")
+
+# The binary caches devices resolve upgrades from. A build is only advertised
+# as available once one of these actually serves its narinfo — see set_available.
+DEV_CACHE = "https://cache.pifinder.eu/pifinder"
+RELEASE_CACHE = "https://cache.pifinder.eu/pifinder-release"
+DEFAULT_CACHES = (DEV_CACHE, RELEASE_CACHE)
 EMPTY_MANIFEST = {
     "schema": 1,
     "generated_at": None,
@@ -49,19 +57,63 @@ def valid_store_path(value: str | None) -> bool:
     return isinstance(value, str) and STORE_PATH_RE.fullmatch(value) is not None
 
 
-def set_available(entry: dict) -> dict:
-    if valid_store_path(entry.get("store_path")):
-        entry["available"] = True
-        entry.pop("reason", None)
-    else:
+def _narinfo_url(store_path: str, cache: str) -> str:
+    digest = Path(store_path).name.split("-", 1)[0]
+    return f"{cache.rstrip('/')}/{digest}.narinfo"
+
+
+def cache_serves(store_path: str, caches, timeout: int = 15) -> bool | None:
+    """Whether the closure is actually downloadable from a cache right now.
+
+    Returns True if any cache serves the narinfo (200); False only if every
+    cache answered and none had it (a real 404 everywhere); None if a cache
+    could not be reached, so presence is unknown. Publishing treats anything
+    other than True as "do not advertise", so a failed push or a down cache can
+    never advertise a build the device cannot fetch.
+    """
+    saw_404 = False
+    for cache in caches:
+        url = _narinfo_url(store_path, cache)
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as resp:
+                if resp.status == 200:
+                    return True
+        except urllib.error.HTTPError as exc:
+            if exc.code == 404:
+                saw_404 = True
+            else:
+                return None
+        except (urllib.error.URLError, OSError, TimeoutError):
+            return None
+    return False if saw_404 else None
+
+
+def set_available(entry: dict, caches=DEFAULT_CACHES, verify: bool = True) -> dict:
+    store_path = entry.get("store_path")
+    if not valid_store_path(store_path):
         entry["store_path"] = None
         entry["available"] = False
         entry.setdefault("reason", "no build")
+        return entry
+
+    if verify:
+        present = cache_serves(store_path, caches)
+        if present is not True:
+            entry["available"] = False
+            entry["reason"] = "not in cache" if present is False else "cache unreachable"
+            return entry
+
+    entry["available"] = True
+    entry.pop("reason", None)
     return entry
 
 
 def replace_entry(entries: list[dict], predicate, entry: dict) -> list[dict]:
-    return [item for item in entries if not predicate(item)] + [entry]
+    # Newest first: consumers (release channels in the update UI, and the
+    # migration target lookup) take the head of the list as the current entry.
+    # The unstable channel is re-sorted after this, so prepending is neutral
+    # there.
+    return [entry] + [item for item in entries if not predicate(item)]
 
 
 def sort_unstable(entries: list[dict]) -> list[dict]:
@@ -75,11 +127,17 @@ def key(item: dict) -> tuple[int, int, str]:
     return sorted(entries, key=key)
 
 
+def resolve_verify(args: argparse.Namespace) -> tuple[tuple[str, ...], bool]:
+    caches = tuple(args.verify_cache) if args.verify_cache else DEFAULT_CACHES
+    return caches, not args.skip_cache_check
+
+
 def update_build(args: argparse.Namespace) -> None:
     manifest = load_manifest(args.manifest)
     channels = manifest["channels"]
     store_path = args.store_path or None
     short_sha = (args.head_sha or args.sha)[:7]
+    caches, verify = resolve_verify(args)
 
     if args.pr_number:
         number = int(args.pr_number)
@@ -95,7 +153,7 @@ def update_build(args: argparse.Namespace) -> None:
             "version": args.version or f"PR#{number}-{short_sha}",
             "store_path": store_path,
         }
-        set_available(entry)
+        set_available(entry, caches, verify)
         channels["unstable"] = replace_entry(
             channels["unstable"],
             lambda item: item.get("kind") == "pr"
@@ -114,7 +172,7 @@ def update_build(args: argparse.Namespace) -> None:
             "version": args.version or f"{args.ref_name}-{short_sha}",
             "store_path": store_path,
         }
-        set_available(entry)
+        set_available(entry, caches, verify)
         channels["unstable"] = replace_entry(
             channels["unstable"],
             lambda item: item.get("kind") == "trunk"
@@ -129,6 +187,7 @@ def update_build(args: argparse.Namespace) -> None:
 
 def update_release(args: argparse.Namespace) -> None:
     manifest = load_manifest(args.manifest)
+    caches, verify = resolve_verify(args)
     channel = "beta" if args.release_type == "beta" else "stable"
     entry = {
         "kind": "release",
@@ -140,8 +199,10 @@ def update_release(args: argparse.Namespace) -> None:
         "source_sha": args.sha,
         "version": args.version,
         "store_path": args.store_path or None,
+        "migration_url": args.migration_url or None,
+        "migration_sha256_url": args.migration_sha256_url or None,
     }
-    set_available(entry)
+    set_available(entry, caches, verify)
     manifest["channels"][channel] = replace_entry(
         manifest["channels"][channel],
         lambda item: item.get("kind") == "release" and item.get("label") == args.tag,
@@ -150,6 +211,17 @@ def update_release(args: argparse.Namespace) -> None:
     save_manifest(args.manifest, manifest)
 
 
+def add_cache_args(sub: argparse.ArgumentParser) -> None:
+    # A build is advertised as available only if one of these caches actually
+    # serves its narinfo. Repeatable; defaults to the dev + release caches.
+    sub.add_argument("--verify-cache", action="append", metavar="URL")
+    sub.add_argument(
+        "--skip-cache-check",
+        action="store_true",
+        help="advertise on store-path syntax alone (tests / offline only)",
+    )
+
+
 def parser() -> argparse.ArgumentParser:
     root = argparse.ArgumentParser()
     sub = root.add_subparsers(dest="command", required=True)
@@ -167,6 +239,7 @@ def parser() -> argparse.ArgumentParser:
     build.add_argument("--head-repo")
     build.add_argument("--head-ref")
     build.add_argument("--head-sha")
+    add_cache_args(build)
     build.set_defaults(func=update_build)
 
     release = sub.add_parser("release")
@@ -177,8 +250,11 @@ def parser() -> argparse.ArgumentParser:
     release.add_argument("--version", required=True)
     release.add_argument("--release-type", choices=("stable", "beta"), required=True)
     release.add_argument("--store-path", required=True)
+    release.add_argument("--migration-url")
+    release.add_argument("--migration-sha256-url")
     release.add_argument("--title")
     release.add_argument("--notes")
+    add_cache_args(release)
     release.set_defaults(func=update_release)
 
     return root

.github/workflows/nixos-pr-build.yml

@@ -55,18 +55,22 @@ jobs:
           attic login pifinder https://cache.pifinder.eu "$ATTIC_TOKEN"
           attic use pifinder:pifinder
 
-      - name: Build NixOS system closure
+      - name: Build NixOS system and on-device dev shell closures
         id: build
         run: |
           nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
             -L --no-link
+          nix build .#devShells.aarch64-linux.default -L --no-link
 
       - name: Push to Attic
         id: push
         run: |
           STORE_PATH=$(nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
             --json | jq -r '.[].outputs.out')
+          DEV_SHELL_PATH=$(nix build .#devShells.aarch64-linux.default \
+            --json | jq -r '.[].outputs.out')
           attic push pifinder:pifinder "$STORE_PATH"
+          attic push pifinder:pifinder "$DEV_SHELL_PATH"
           echo "store_path=$STORE_PATH" >> "$GITHUB_OUTPUT"
 
   # Wait up to ~15 min for the native builder, then decide on the hosted fallback.
@@ -135,10 +139,11 @@ jobs:
           attic login pifinder https://cache.pifinder.eu "$ATTIC_TOKEN"
           attic use pifinder:pifinder
 
-      - name: Build NixOS system closure
+      - name: Build NixOS system and on-device dev shell closures
         run: |
           nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
             -L --no-link
+          nix build .#devShells.aarch64-linux.default -L --no-link
 
       - name: Push to Attic
         id: push
@@ -147,9 +152,12 @@ jobs:
         run: |
           STORE_PATH=$(nix build .#nixosConfigurations.pifinder.config.system.build.toplevel \
             --json | jq -r '.[].outputs.out')
+          DEV_SHELL_PATH=$(nix build .#devShells.aarch64-linux.default \
+            --json | jq -r '.[].outputs.out')
           echo "store_path=$STORE_PATH" >> "$GITHUB_OUTPUT"
           if [ -n "$ATTIC_TOKEN" ]; then
             attic push pifinder:pifinder "$STORE_PATH"
+            attic push pifinder:pifinder "$DEV_SHELL_PATH"
           else
             echo "No ATTIC_TOKEN — skipping push; build is verify-only"
           fi

.github/workflows/nox.yml

@@ -1,4 +1,9 @@
 name: nox
+# The nixos branch manages the Python environment with uv (pyproject.toml +
+# uv.lock) instead of nox + requirements*.txt, so this runs the same checks
+# (ruff, mypy, pytest) through uv. The workflow keeps upstream's "nox" name so
+# PR check wiring stays identical across branches.
+#
 # Run on pushes to the long-lived branches and on pull requests. Restricting
 # `push` to main/release means a push to a feature branch with an open PR only
 # triggers the `pull_request` run (not also a `push` run), so a PR gets a single
@@ -18,9 +23,21 @@ jobs:
         working-directory: ./python
     steps:
       - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
         with:
-          submodules: true
-      - uses: wntrblm/nox@2024.04.15
-        with:
-          python-versions: "3.9"
-      - run: nox -s lint format type_hints smoke_tests unit_tests ui_tests
+          enable-cache: true
+      # Same ruff pin as upstream main's noxfile, so shared files keep one style.
+      - name: Lint
+        run: uvx ruff@0.4.8 check --config "builtins=['_']" .
+      - name: Format check
+        run: uvx ruff@0.4.8 format --check .
+      - name: Sync environment
+        run: uv sync --frozen
+      - name: Type check
+        run: uv run mypy PiFinder
+      - name: Smoke tests
+        run: uv run pytest -m smoke
+      - name: Unit tests
+        run: uv run pytest -m unit
+      - name: UI module tests
+        run: uv run pytest -m integration tests/test_ui_modules.py