diff --git a/docs/deepops/validation.md b/docs/deepops/validation.md new file mode 100644 index 000000000..91b70c872 --- /dev/null +++ b/docs/deepops/validation.md @@ -0,0 +1,95 @@ +# Validating a DeepOps deployment + +DeepOps ships small validation tools that answer one question with a stable, +machine-readable contract: **did the deployment work?** They are safe to run +repeatedly, designed for both humans and automation (including AI agents), +and complement the heavier workload tests under `workloads/`. + +All three tools: + +- print one human-readable line per check by default, or a single flat JSON + object with `--json` +- exit `0` when every check passes, `1` when any check fails, and `2` on + usage or environment errors +- fail loudly with a `failures` list explaining exactly what is wrong + +## Preflight: `scripts/validation/deepops_doctor.py` + +Run from the DeepOps repository root on the provisioning machine before +running cluster playbooks: + +```bash +python3 scripts/validation/deepops_doctor.py +python3 scripts/validation/deepops_doctor.py --remote # adds SSH checks +python3 scripts/validation/deepops_doctor.py --json +``` + +Local checks: Ansible present, Galaxy dependencies installed, the Kubespray +submodule initialized, the configuration directory present, and the inventory +parseable with at least one host. With `--remote` it also verifies host +reachability (`ansible -m ping`), reports which hosts have NVIDIA PCI +devices, and reports sshd systemd overrides (see the GPU visibility note +below). + +## Slurm: `scripts/validation/validate_slurm.py` + +Run on any Slurm cluster node after `playbooks/slurm-cluster.yml`: + +```bash +python3 scripts/validation/validate_slurm.py +python3 scripts/validation/validate_slurm.py --json +python3 scripts/validation/validate_slurm.py --skip-gpu-job # read-only +``` + +Checks controller reachability, node availability, configured `gres/gpu` +resources, and (by default) submits a single-GPU `srun` job running +`nvidia-smi`. Example JSON: + +```json +{ + "ok": true, + "slurm_version": "26.05.1", + "controller_reachable": true, + "nodes_total": 1, + "nodes_available": 1, + "nodes_unavailable": 0, + "node_states": {"idle": 1}, + "gpus_configured": 1, + "gpu_job_ran": true, + "gpu_job_ok": true, + "gpus_visible_in_job": 1, + "failures": [] +} +``` + +### GPU visibility note + +On DeepOps Slurm clusters, GPUs are hidden from ordinary SSH sessions on +cluster nodes; `nvidia-smi` outside a Slurm job may report +`No devices were found` even when the GPUs and driver are healthy. This is +expected behavior, not a failure. The authoritative check is the `srun` job +this script runs. + +## Kubernetes: `scripts/validation/validate_k8s.py` + +Run anywhere `kubectl` reaches the cluster after `playbooks/k8s-cluster.yml`: + +```bash +python3 scripts/validation/validate_k8s.py +python3 scripts/validation/validate_k8s.py --cuda-smoke +python3 scripts/validation/validate_k8s.py --json --cuda-smoke +``` + +Checks API reachability, node readiness, allocatable `nvidia.com/gpu` +resources, and GPU Operator pod health. With `--cuda-smoke` it creates a +temporary `deepops-validate` namespace, runs a CUDA pod that requests one GPU +and lists it with `nvidia-smi -L`, and deletes the namespace on success (the +namespace is kept on failure for debugging). + +For an exhaustive every-GPU job test, see `scripts/k8s/verify_gpu.sh`. + +## Testing the tools + +```bash +python3 -m unittest discover -s scripts/validation/tests +``` diff --git a/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc b/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc new file mode 100644 index 000000000..b92ba6ba1 Binary files /dev/null and b/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc differ diff --git a/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc b/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc new file mode 100644 index 000000000..7eb449bd7 Binary files /dev/null and b/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc differ diff --git a/scripts/validation/__pycache__/validate_slurm.cpython-312.pyc b/scripts/validation/__pycache__/validate_slurm.cpython-312.pyc new file mode 100644 index 000000000..ca6b456e4 Binary files /dev/null and b/scripts/validation/__pycache__/validate_slurm.cpython-312.pyc differ diff --git a/scripts/validation/deepops_doctor.py b/scripts/validation/deepops_doctor.py new file mode 100755 index 000000000..124058a12 --- /dev/null +++ b/scripts/validation/deepops_doctor.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Preflight checks for a DeepOps provisioning environment. + +Run this from the DeepOps repository root on the provisioning machine before +running cluster playbooks. It verifies the local environment (Ansible, Galaxy +dependencies, Kubespray submodule, configuration directory, inventory) and, +with ``--remote``, host reachability and GPU visibility over the configured +inventory. + +The default output is one line per check. With ``--json`` the script prints a +single JSON object with a stable ``checks`` list so automation and AI agents +can consume the result directly. + +Exit codes: 0 = all checks passed, 1 = one or more checks failed, +2 = not run from a DeepOps repository root. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys + + +def run(cmd, timeout=120, env=None): + """Run a command, returning (rc, stdout, stderr) without raising.""" + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, env=env + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + except subprocess.TimeoutExpired: + return 124, "", "timeout after %ss" % timeout + except FileNotFoundError: + return 127, "", "command not found: %s" % cmd[0] + + +def check(checks, name, ok, detail): + checks.append({"name": name, "ok": bool(ok), "detail": detail}) + return ok + + +def count_positive_stdout_hosts(output): + """Count hosts whose ansible one-line ``(stdout) N`` value is a positive int.""" + hosts = 0 + for line in output.splitlines(): + if "(stdout)" not in line: + continue + tail = line.rsplit("(stdout)", 1)[1].strip() + first = tail.split("\\n")[0].strip() + try: + if int(first) > 0: + hosts += 1 + except ValueError: + continue + return hosts + + +def count_inventory_hosts(inventory_json): + """Count hosts and detect DeepOps groups in ``ansible-inventory --list`` output.""" + hosts = set() + meta = inventory_json.get("_meta", {}).get("hostvars", {}) + hosts.update(meta.keys()) + for group, data in inventory_json.items(): + if group == "_meta" or not isinstance(data, dict): + continue + hosts.update(data.get("hosts", [])) + groups = [g for g in inventory_json if g not in ("_meta", "all", "ungrouped")] + return len(hosts), sorted(groups) + + +def main(): + parser = argparse.ArgumentParser( + description="Preflight checks for a DeepOps provisioning environment." + ) + parser.add_argument("--json", action="store_true", help="emit one JSON object") + parser.add_argument( + "--inventory", + default="", + help="inventory path (default: config/inventory via ansible.cfg)", + ) + parser.add_argument( + "--remote", + action="store_true", + help="also check host reachability and GPU visibility over SSH", + ) + args = parser.parse_args() + + root = os.getcwd() + if not os.path.exists(os.path.join(root, "ansible.cfg")) or not os.path.isdir( + os.path.join(root, "playbooks") + ): + print( + "error: run from the DeepOps repository root (ansible.cfg not found)", + file=sys.stderr, + ) + return 2 + + checks = [] + + rc, out, _ = run(["ansible", "--version"], timeout=60) + ansible_ok = check( + checks, + "ansible_installed", + rc == 0, + out.splitlines()[0] if rc == 0 and out else "install Ansible via ./scripts/setup.sh", + ) + check( + checks, + "ansible_playbook_installed", + shutil.which("ansible-playbook") is not None, + "ansible-playbook on PATH" if shutil.which("ansible-playbook") else "missing ansible-playbook", + ) + + galaxy_marker = os.path.join(root, "roles", "galaxy") + check( + checks, + "galaxy_dependencies_installed", + os.path.isdir(galaxy_marker) and bool(os.listdir(galaxy_marker)), + "roles/galaxy populated" + if os.path.isdir(galaxy_marker) and os.listdir(galaxy_marker) + else "run ./scripts/setup.sh to install Ansible Galaxy requirements", + ) + + kubespray_marker = os.path.join(root, "submodules", "kubespray", "cluster.yml") + check( + checks, + "kubespray_submodule_initialized", + os.path.exists(kubespray_marker), + "submodules/kubespray present" + if os.path.exists(kubespray_marker) + else "run: git submodule update --init --recursive", + ) + + config_dir = os.environ.get("DEEPOPS_CONFIG_DIR", os.path.join(root, "config")) + config_ok = check( + checks, + "config_dir_exists", + os.path.isdir(config_dir), + config_dir + if os.path.isdir(config_dir) + else "copy config.example/ to config/ and edit the inventory", + ) + + inventory = args.inventory or os.path.join(config_dir, "inventory") + hosts_total = 0 + groups = [] + if ansible_ok and config_ok and os.path.exists(inventory): + rc, out, err = run( + ["ansible-inventory", "-i", inventory, "--list"], timeout=120 + ) + parsed_ok = False + if rc == 0: + try: + hosts_total, groups = count_inventory_hosts(json.loads(out)) + parsed_ok = True + except json.JSONDecodeError: + # Leave parsed_ok False; the inventory_parses check below + # reports the failure with the ansible-inventory context. + parsed_ok = False + check( + checks, + "inventory_parses", + parsed_ok, + "%d host(s), groups: %s" % (hosts_total, ", ".join(groups)) + if parsed_ok + else "ansible-inventory failed: %s" % (err or "unparseable output"), + ) + if parsed_ok: + check( + checks, + "inventory_has_hosts", + hosts_total > 0, + "%d host(s) defined" % hosts_total + if hosts_total + else "inventory defines no hosts", + ) + else: + check( + checks, + "inventory_parses", + False, + "inventory not found at %s" % inventory, + ) + + if args.remote and hosts_total > 0: + rc, out, err = run( + ["ansible", "all", "-i", inventory, "-m", "ping", "-o"], timeout=300 + ) + reachable = out.count("SUCCESS") + check( + checks, + "hosts_reachable", + rc == 0, + "%d/%d host(s) reachable" % (reachable, hosts_total), + ) + + rc, out, _ = run( + [ + "ansible", "all", "-i", inventory, "-m", "shell", "-o", + "-a", "lspci 2>/dev/null | grep -ci nvidia || true", + ], + timeout=300, + ) + gpu_hosts = count_positive_stdout_hosts(out) if rc == 0 else 0 + check( + checks, + "gpus_detected_on_hosts", + True, + "%d host(s) report NVIDIA PCI devices (informational)" % gpu_hosts, + ) + + rc, out, _ = run( + [ + "ansible", "all", "-i", inventory, "-m", "shell", "-o", + "-a", "systemctl show ssh.service sshd.service -p DeviceAllow 2>/dev/null | grep -ci nvidiactl || true", + ], + timeout=300, + ) + overrides = count_positive_stdout_hosts(out) if rc == 0 else 0 + check( + checks, + "ssh_gpu_visibility_override", + True, + "%d host(s) restrict GPU device access for SSH sessions (Slurm login " + "GPU hiding); on those hosts direct nvidia-smi over SSH is expected to " + "fail and srun is the authoritative GPU test" % overrides, + ) + + ok = all(c["ok"] for c in checks) + result = {"ok": ok, "checks": checks} + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for c in checks: + print("%s %s: %s" % ("PASS" if c["ok"] else "FAIL", c["name"], c["detail"])) + print("ok=%s" % ok) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc b/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc new file mode 100644 index 000000000..7215da80f Binary files /dev/null and b/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc differ diff --git a/scripts/validation/tests/test_validation_scripts.py b/scripts/validation/tests/test_validation_scripts.py new file mode 100644 index 000000000..3010a5bfe --- /dev/null +++ b/scripts/validation/tests/test_validation_scripts.py @@ -0,0 +1,126 @@ +"""Unit tests for the validation script parsers (standard library only). + +Run with: python3 -m unittest discover scripts/validation/tests +""" + +import importlib.util +import os +import unittest + +SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def load(name): + spec = importlib.util.spec_from_file_location( + name, os.path.join(SCRIPTS_DIR, name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +validate_slurm = load("validate_slurm") +validate_k8s = load("validate_k8s") +deepops_doctor = load("deepops_doctor") + + +class TestSlurmParsers(unittest.TestCase): + def test_sinfo_states_healthy(self): + out = "node1 idle\nnode2 mixed\nnode3 allocated\n" + total, avail, unavail, states = validate_slurm.parse_sinfo_states(out) + self.assertEqual((total, avail, unavail), (3, 3, 0)) + self.assertEqual(states["idle"], 1) + + def test_sinfo_states_down_and_flags(self): + out = "node1 idle\nnode2 down*\nnode3 drained\n" + total, avail, unavail, states = validate_slurm.parse_sinfo_states(out) + self.assertEqual((total, avail, unavail), (3, 1, 2)) + self.assertIn("down", states) + self.assertIn("drained", states) + + def test_sinfo_states_dedupes_partition_overlap(self): + out = "node1 idle\nnode1 idle\n" + total, _, _, _ = validate_slurm.parse_sinfo_states(out) + self.assertEqual(total, 1) + + def test_gres_gpu_totals(self): + out = "node1 gpu:4\nnode2 gpu:h100:8(S:0-1)\nnode3 (null)\n" + self.assertEqual(validate_slurm.parse_gres_gpus(out), 12) + + def test_gres_dedupes_nodes(self): + out = "node1 gpu:4\nnode1 gpu:4\n" + self.assertEqual(validate_slurm.parse_gres_gpus(out), 4) + + +class TestK8sParsers(unittest.TestCase): + def test_summarize_nodes(self): + doc = { + "items": [ + { + "status": { + "conditions": [{"type": "Ready", "status": "True"}], + "allocatable": {"nvidia.com/gpu": "8"}, + } + }, + { + "status": { + "conditions": [{"type": "Ready", "status": "False"}], + "allocatable": {}, + } + }, + ] + } + total, ready, gpus = validate_k8s.summarize_nodes(doc) + self.assertEqual((total, ready, gpus), (2, 1, 8)) + + def test_summarize_gpu_pods(self): + doc = { + "items": [ + { + "status": { + "phase": "Running", + "containerStatuses": [{"ready": True}], + } + }, + { + "status": { + "phase": "Running", + "containerStatuses": [{"ready": False}], + } + }, + {"status": {"phase": "Succeeded"}}, + {"status": {"phase": "Pending"}}, + ] + } + total, ready = validate_k8s.summarize_gpu_pods(doc) + self.assertEqual((total, ready), (4, 2)) + + def test_smoke_pod_manifest_requests_one_gpu(self): + pod = validate_k8s.smoke_pod_manifest("example/image:tag") + limits = pod["spec"]["containers"][0]["resources"]["limits"] + self.assertEqual(limits["nvidia.com/gpu"], 1) + + +class TestDoctorParsers(unittest.TestCase): + def test_count_inventory_hosts(self): + doc = { + "_meta": {"hostvars": {"n1": {}, "n2": {}}}, + "all": {"children": ["slurm-master"]}, + "slurm-master": {"hosts": ["n1"]}, + "slurm-node": {"hosts": ["n2"]}, + } + hosts, groups = deepops_doctor.count_inventory_hosts(doc) + self.assertEqual(hosts, 2) + self.assertIn("slurm-node", groups) + + def test_count_positive_stdout_hosts(self): + out = ( + "n1 | CHANGED | rc=0 | (stdout) 2\n" + "n2 | CHANGED | rc=0 | (stdout) 0\n" + "n3 | CHANGED | rc=0 | (stdout) not-a-number\n" + ) + self.assertEqual(deepops_doctor.count_positive_stdout_hosts(out), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validation/validate_k8s.py b/scripts/validation/validate_k8s.py new file mode 100755 index 000000000..98808b44f --- /dev/null +++ b/scripts/validation/validate_k8s.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Validate a DeepOps Kubernetes GPU deployment with a machine-readable verdict. + +Run this anywhere ``kubectl`` can reach the cluster after deploying +``playbooks/k8s-cluster.yml``. It checks node readiness, allocatable +``nvidia.com/gpu`` resources, GPU stack pod health, and can optionally run a +CUDA smoke pod that requests one GPU. + +The default output is one human-readable line per check. With ``--json`` the +script prints a single flat JSON object with stable field names so automation +and AI agents can consume the result directly. + +Exit codes: 0 = all checks passed, 1 = one or more checks failed, +2 = usage or environment error (kubectl not found). +""" + +import argparse +import json +import shutil +import subprocess +import sys +import time + +SMOKE_NAMESPACE = "deepops-validate" +SMOKE_POD = "deepops-validate-cuda" + + +def run(cmd, timeout=60, input_text=None): + """Run a command, returning (rc, stdout, stderr) without raising.""" + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, input=input_text + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + except subprocess.TimeoutExpired: + return 124, "", "timeout after %ss" % timeout + except FileNotFoundError: + return 127, "", "command not found: %s" % cmd[0] + + +def summarize_nodes(nodes_json): + """Summarize a ``kubectl get nodes -o json`` document. + + Returns (nodes_total, nodes_ready, gpus_allocatable). + """ + total = ready = gpus = 0 + for item in nodes_json.get("items", []): + total += 1 + for cond in item.get("status", {}).get("conditions", []): + if cond.get("type") == "Ready" and cond.get("status") == "True": + ready += 1 + break + alloc = item.get("status", {}).get("allocatable", {}) + try: + gpus += int(alloc.get("nvidia.com/gpu", "0")) + except ValueError: + # A malformed allocatable value counts as zero GPUs; the + # gpus_allocatable check will then fail loudly for this cluster. + continue + return total, ready, gpus + + +def summarize_gpu_pods(pods_json): + """Summarize GPU stack pods. Returns (pods_total, pods_ready).""" + total = ready = 0 + for item in pods_json.get("items", []): + total += 1 + phase = item.get("status", {}).get("phase", "") + if phase == "Succeeded": + ready += 1 + continue + if phase != "Running": + continue + statuses = item.get("status", {}).get("containerStatuses", []) + if statuses and all(s.get("ready") for s in statuses): + ready += 1 + return total, ready + + +def smoke_pod_manifest(image): + return { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": SMOKE_POD, "namespace": SMOKE_NAMESPACE}, + "spec": { + "restartPolicy": "Never", + "containers": [ + { + "name": "cuda-smoke", + "image": image, + "command": ["nvidia-smi", "-L"], + "resources": {"limits": {"nvidia.com/gpu": 1}}, + } + ], + }, + } + + +def run_cuda_smoke(result, image, timeout): + result["cuda_smoke_ran"] = True + run(["kubectl", "delete", "namespace", SMOKE_NAMESPACE, "--ignore-not-found"], timeout=120) + rc, _, err = run(["kubectl", "create", "namespace", SMOKE_NAMESPACE]) + if rc != 0: + result["failures"].append("could not create smoke namespace: %s" % err) + return + rc, _, err = run( + ["kubectl", "apply", "-f", "-"], + input_text=json.dumps(smoke_pod_manifest(image)), + ) + if rc != 0: + result["failures"].append("could not create smoke pod: %s" % err) + return + deadline = time.time() + timeout + phase = "" + while time.time() < deadline: + rc, phase, _ = run( + ["kubectl", "-n", SMOKE_NAMESPACE, "get", "pod", SMOKE_POD, + "-o", "jsonpath={.status.phase}"] + ) + if phase in ("Succeeded", "Failed"): + break + time.sleep(5) + rc, logs, _ = run(["kubectl", "-n", SMOKE_NAMESPACE, "logs", SMOKE_POD]) + if phase == "Succeeded" and rc == 0 and "GPU" in logs: + result["cuda_smoke_ok"] = True + result["cuda_smoke_gpus"] = sum( + 1 for line in logs.splitlines() if line.startswith("GPU ") + ) + run(["kubectl", "delete", "namespace", SMOKE_NAMESPACE, "--ignore-not-found"], timeout=120) + else: + result["failures"].append( + "CUDA smoke pod did not succeed (phase=%s); namespace %s kept for debugging" + % (phase or "unknown", SMOKE_NAMESPACE) + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Validate a DeepOps Kubernetes GPU deployment." + ) + parser.add_argument("--json", action="store_true", help="emit one JSON object") + parser.add_argument( + "--cuda-smoke", + action="store_true", + help="run a CUDA pod requesting one GPU (creates and deletes namespace %s)" + % SMOKE_NAMESPACE, + ) + parser.add_argument( + "--cuda-image", + default="nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04", + help="image for the CUDA smoke pod", + ) + parser.add_argument( + "--smoke-timeout", + type=int, + default=600, + help="seconds to wait for the CUDA smoke pod (default: 600)", + ) + parser.add_argument( + "--allow-no-gpus", + action="store_true", + help="do not fail when the cluster has no allocatable GPUs", + ) + args = parser.parse_args() + + if not shutil.which("kubectl"): + print("error: kubectl not found in PATH", file=sys.stderr) + return 2 + + result = { + "ok": False, + "api_reachable": False, + "nodes_total": 0, + "nodes_ready": 0, + "gpus_allocatable": 0, + "gpu_stack_pods_total": 0, + "gpu_stack_pods_ready": 0, + "cuda_smoke_ran": False, + "cuda_smoke_ok": False, + "cuda_smoke_gpus": 0, + "failures": [], + } + + rc, out, err = run(["kubectl", "get", "nodes", "-o", "json"], timeout=120) + if rc != 0: + result["failures"].append("kubectl get nodes failed: %s" % (err or "rc=%s" % rc)) + else: + result["api_reachable"] = True + try: + total, ready, gpus = summarize_nodes(json.loads(out)) + except json.JSONDecodeError: + result["failures"].append("could not parse kubectl node output") + total = ready = gpus = 0 + result["nodes_total"] = total + result["nodes_ready"] = ready + result["gpus_allocatable"] = gpus + if total == 0: + result["failures"].append("cluster reports zero nodes") + elif ready < total: + result["failures"].append( + "%d of %d nodes are not Ready" % (total - ready, total) + ) + if gpus == 0 and not args.allow_no_gpus: + result["failures"].append( + "no allocatable nvidia.com/gpu resources; check GPU Operator or device plugin" + ) + + if result["api_reachable"]: + rc, out, _ = run( + ["kubectl", "get", "pods", "--all-namespaces", + "-l", "app.kubernetes.io/managed-by in (gpu-operator)", "-o", "json"], + timeout=120, + ) + pods_total = pods_ready = 0 + if rc == 0: + try: + pods_total, pods_ready = summarize_gpu_pods(json.loads(out)) + except json.JSONDecodeError: + # Best-effort check: unparseable pod output leaves the counts + # at zero; the authoritative GPU signal is gpus_allocatable. + pods_total = pods_ready = 0 + result["gpu_stack_pods_total"] = pods_total + result["gpu_stack_pods_ready"] = pods_ready + if pods_total and pods_ready < pods_total: + result["failures"].append( + "%d of %d GPU stack pods are not ready" % (pods_total - pods_ready, pods_total) + ) + + if args.cuda_smoke and result["api_reachable"] and result["gpus_allocatable"] > 0: + run_cuda_smoke(result, args.cuda_image, args.smoke_timeout) + + result["ok"] = not result["failures"] + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for key in ( + "api_reachable", + "nodes_total", + "nodes_ready", + "gpus_allocatable", + "gpu_stack_pods_total", + "gpu_stack_pods_ready", + "cuda_smoke_ran", + "cuda_smoke_ok", + "cuda_smoke_gpus", + ): + print("%s=%s" % (key, result[key])) + for failure in result["failures"]: + print("FAIL: %s" % failure) + print("ok=%s" % result["ok"]) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/validate_slurm.py b/scripts/validation/validate_slurm.py new file mode 100755 index 000000000..c7006e888 --- /dev/null +++ b/scripts/validation/validate_slurm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Validate a DeepOps Slurm deployment and report a machine-readable verdict. + +Run this on a Slurm controller, login, or compute node after deploying +``playbooks/slurm-cluster.yml``. It checks controller reachability, node +health, GPU (GRES) configuration, and optionally runs a single-GPU job. + +The default output is one human-readable line per check. With ``--json`` the +script prints a single flat JSON object with stable field names so automation +and AI agents can consume the result directly. + +Exit codes: 0 = all checks passed, 1 = one or more checks failed, +2 = usage or environment error (Slurm commands not found). + +Note: on DeepOps Slurm clusters, ``nvidia-smi`` in an ordinary SSH session on +a login/compute node may report "No devices were found" because GPUs are +hidden outside Slurm-managed jobs. That is expected; the authoritative GPU +check is the ``srun`` job this script runs. +""" + +import argparse +import json +import re +import shutil +import subprocess +import sys + + +def run(cmd, timeout=60): + """Run a command, returning (rc, stdout, stderr) without raising.""" + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + except subprocess.TimeoutExpired: + return 124, "", "timeout after %ss" % timeout + except FileNotFoundError: + return 127, "", "command not found: %s" % cmd[0] + + +def parse_sinfo_states(output): + """Parse ``sinfo -h -N -o '%n %T'`` output into node-state counts. + + Returns (nodes_total, nodes_available, nodes_unavailable, states) where + states maps state name (lowercase, trailing '*' stripped) to a count. + """ + states = {} + seen = set() + for line in output.splitlines(): + parts = line.split() + if len(parts) != 2: + continue + node, state = parts + if node in seen: + continue + seen.add(node) + state = state.strip().rstrip("*+~#!%$@^-").lower() + states[state] = states.get(state, 0) + 1 + available_states = ("idle", "mixed", "allocated", "completing") + available = sum(states.get(s, 0) for s in available_states) + total = len(seen) + return total, available, total - available, states + + +def parse_gres_gpus(output): + """Parse ``sinfo -h -N -o '%n %G'`` output into a total GPU count.""" + total = 0 + seen = set() + for line in output.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2 or parts[0] in seen: + continue + seen.add(parts[0]) + for match in re.finditer(r"gpu(?::[^:,(\s]+)?:(\d+)", parts[1]): + total += int(match.group(1)) + return total + + +def main(): + parser = argparse.ArgumentParser( + description="Validate a DeepOps Slurm deployment." + ) + parser.add_argument("--json", action="store_true", help="emit one JSON object") + parser.add_argument( + "--skip-gpu-job", + action="store_true", + help="skip the srun single-GPU test job", + ) + parser.add_argument( + "--allow-unavailable-nodes", + action="store_true", + help="do not fail when some nodes are down, drained, or unknown", + ) + parser.add_argument( + "--partition", + default="", + help="partition for the GPU test job (default: cluster default)", + ) + parser.add_argument( + "--gpu-job-timeout", + type=int, + default=300, + help="seconds to wait for the GPU test job (default: 300)", + ) + args = parser.parse_args() + + if not shutil.which("sinfo"): + print("error: sinfo not found; run this on a Slurm node", file=sys.stderr) + return 2 + + result = { + "ok": False, + "slurm_version": "", + "controller_reachable": False, + "nodes_total": 0, + "nodes_available": 0, + "nodes_unavailable": 0, + "node_states": {}, + "gpus_configured": 0, + "gpu_job_ran": False, + "gpu_job_ok": False, + "gpus_visible_in_job": 0, + "failures": [], + } + + rc, out, _ = run(["sinfo", "--version"]) + if rc == 0 and out: + result["slurm_version"] = out.split()[-1] + + rc, out, err = run(["sinfo", "-h", "-N", "-o", "%n %T"]) + if rc != 0: + result["failures"].append("sinfo failed: %s" % (err or "rc=%s" % rc)) + else: + result["controller_reachable"] = True + total, avail, unavail, states = parse_sinfo_states(out) + result["nodes_total"] = total + result["nodes_available"] = avail + result["nodes_unavailable"] = unavail + result["node_states"] = states + if total == 0: + result["failures"].append("no nodes are defined in Slurm") + if unavail and not args.allow_unavailable_nodes: + result["failures"].append( + "%d node(s) unavailable: %s" + % (unavail, ", ".join(sorted(k for k in states if k not in ("idle", "mixed", "allocated", "completing")))) + ) + + rc, out, _ = run(["sinfo", "-h", "-N", "-o", "%n %G"]) + if rc == 0: + result["gpus_configured"] = parse_gres_gpus(out) + if result["controller_reachable"] and result["gpus_configured"] == 0: + result["failures"].append("no GPUs (gres/gpu) configured on any node") + + if not args.skip_gpu_job and result["controller_reachable"] and result["gpus_configured"] > 0: + result["gpu_job_ran"] = True + cmd = ["srun", "--gpus=1", "--time=5"] + if args.partition: + cmd += ["--partition", args.partition] + cmd += ["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"] + rc, out, err = run(cmd, timeout=args.gpu_job_timeout) + if rc == 0 and out: + result["gpu_job_ok"] = True + result["gpus_visible_in_job"] = len(out.splitlines()) + else: + result["failures"].append( + "srun GPU job failed: %s" % (err or out or "rc=%s" % rc) + ) + + result["ok"] = not result["failures"] + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for key in ( + "slurm_version", + "controller_reachable", + "nodes_total", + "nodes_available", + "nodes_unavailable", + "gpus_configured", + "gpu_job_ran", + "gpu_job_ok", + ): + print("%s=%s" % (key, result[key])) + for failure in result["failures"]: + print("FAIL: %s" % failure) + print("ok=%s" % result["ok"]) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main())