From 22a211959d077afc6f673a426b059614e2b8bcea Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 13 Jul 2026 14:46:40 -0600 Subject: [PATCH 1/3] Add machine-readable deployment validation tools Add scripts/validation/ with three standard-library Python tools that report a stable machine-readable verdict for automation and AI agents: - deepops_doctor.py: provisioning-environment preflight (Ansible, Galaxy dependencies, Kubespray submodule, config directory, inventory parse, and optional remote reachability/GPU checks) - validate_slurm.py: Slurm deployment validation (controller, node availability, gres/gpu configuration, single-GPU srun job) - validate_k8s.py: Kubernetes GPU deployment validation (node readiness, allocatable GPUs, GPU Operator pod health, optional CUDA smoke pod) Each prints one line per check or a flat JSON object with --json, exits nonzero on failure with an explicit failures list, and documents the Slurm login GPU-hiding behavior that otherwise reads as a driver failure. Includes stdlib unit tests and docs/deepops/validation.md. --- docs/deepops/validation.md | 95 +++++++ .../deepops_doctor.cpython-312.pyc | Bin 0 -> 10441 bytes .../__pycache__/validate_k8s.cpython-312.pyc | Bin 0 -> 10719 bytes .../validate_slurm.cpython-312.pyc | Bin 0 -> 8561 bytes scripts/validation/deepops_doctor.py | 242 +++++++++++++++++ .../test_validation_scripts.cpython-312.pyc | Bin 0 -> 6810 bytes .../tests/test_validation_scripts.py | 127 +++++++++ scripts/validation/validate_k8s.py | 252 ++++++++++++++++++ scripts/validation/validate_slurm.py | 193 ++++++++++++++ 9 files changed, 909 insertions(+) create mode 100644 docs/deepops/validation.md create mode 100644 scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc create mode 100644 scripts/validation/__pycache__/validate_k8s.cpython-312.pyc create mode 100644 scripts/validation/__pycache__/validate_slurm.cpython-312.pyc create mode 100755 scripts/validation/deepops_doctor.py create mode 100644 scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc create mode 100644 scripts/validation/tests/test_validation_scripts.py create mode 100755 scripts/validation/validate_k8s.py create mode 100755 scripts/validation/validate_slurm.py diff --git a/docs/deepops/validation.md b/docs/deepops/validation.md new file mode 100644 index 000000000..91b70c872 --- /dev/null +++ b/docs/deepops/validation.md @@ -0,0 +1,95 @@ +# Validating a DeepOps deployment + +DeepOps ships small validation tools that answer one question with a stable, +machine-readable contract: **did the deployment work?** They are safe to run +repeatedly, designed for both humans and automation (including AI agents), +and complement the heavier workload tests under `workloads/`. + +All three tools: + +- print one human-readable line per check by default, or a single flat JSON + object with `--json` +- exit `0` when every check passes, `1` when any check fails, and `2` on + usage or environment errors +- fail loudly with a `failures` list explaining exactly what is wrong + +## Preflight: `scripts/validation/deepops_doctor.py` + +Run from the DeepOps repository root on the provisioning machine before +running cluster playbooks: + +```bash +python3 scripts/validation/deepops_doctor.py +python3 scripts/validation/deepops_doctor.py --remote # adds SSH checks +python3 scripts/validation/deepops_doctor.py --json +``` + +Local checks: Ansible present, Galaxy dependencies installed, the Kubespray +submodule initialized, the configuration directory present, and the inventory +parseable with at least one host. With `--remote` it also verifies host +reachability (`ansible -m ping`), reports which hosts have NVIDIA PCI +devices, and reports sshd systemd overrides (see the GPU visibility note +below). + +## Slurm: `scripts/validation/validate_slurm.py` + +Run on any Slurm cluster node after `playbooks/slurm-cluster.yml`: + +```bash +python3 scripts/validation/validate_slurm.py +python3 scripts/validation/validate_slurm.py --json +python3 scripts/validation/validate_slurm.py --skip-gpu-job # read-only +``` + +Checks controller reachability, node availability, configured `gres/gpu` +resources, and (by default) submits a single-GPU `srun` job running +`nvidia-smi`. Example JSON: + +```json +{ + "ok": true, + "slurm_version": "26.05.1", + "controller_reachable": true, + "nodes_total": 1, + "nodes_available": 1, + "nodes_unavailable": 0, + "node_states": {"idle": 1}, + "gpus_configured": 1, + "gpu_job_ran": true, + "gpu_job_ok": true, + "gpus_visible_in_job": 1, + "failures": [] +} +``` + +### GPU visibility note + +On DeepOps Slurm clusters, GPUs are hidden from ordinary SSH sessions on +cluster nodes; `nvidia-smi` outside a Slurm job may report +`No devices were found` even when the GPUs and driver are healthy. This is +expected behavior, not a failure. The authoritative check is the `srun` job +this script runs. + +## Kubernetes: `scripts/validation/validate_k8s.py` + +Run anywhere `kubectl` reaches the cluster after `playbooks/k8s-cluster.yml`: + +```bash +python3 scripts/validation/validate_k8s.py +python3 scripts/validation/validate_k8s.py --cuda-smoke +python3 scripts/validation/validate_k8s.py --json --cuda-smoke +``` + +Checks API reachability, node readiness, allocatable `nvidia.com/gpu` +resources, and GPU Operator pod health. With `--cuda-smoke` it creates a +temporary `deepops-validate` namespace, runs a CUDA pod that requests one GPU +and lists it with `nvidia-smi -L`, and deletes the namespace on success (the +namespace is kept on failure for debugging). + +For an exhaustive every-GPU job test, see `scripts/k8s/verify_gpu.sh`. + +## Testing the tools + +```bash +python3 -m unittest discover -s scripts/validation/tests +``` diff --git a/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc b/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46fc4005a73d21db723134f2aeb3619e0c6b5ba3 GIT binary patch literal 10441 zcmbt4TWlLwb~EG*Uqe!)BOKL(aqA1oW79M3LreXNHnR z4-+O>AkxMQOS_9o+oo6<4Pd2lu?nz2oVIA4w7>=}P>`llaVPrlqG?h8DYh5L=Bwup zIYUXSwbR`Jc<-Hi&pr2?bI&>V&N=*dhr>p|;~9D>WPFAo{tLfo4^?bD`Ux~<2}E2Z z5Qz+7@*=6$hKmNZHeNKUHFc3vYx*Ls)~1UlWJSgwk{8X0`XO=Af@mnM$ONSgnW1EO zJF;A9Aqb(yK=7teowjtThJNP5;_HWGp+OQ8h6#h$#=UPKh%*8|5)KANCDuR6`>%-X zNK9ZkwwLGQr{f|U7h+>UF&K*mqXCwWjs=BSG{Q%vcAM>7BFajmL6{-LBGBUXNdg~_ zi9snQOt3;MCb2OL$gjs0;rye)D9>KzVL{#|B%l^c~M0Zj`_LpT3XnauBaHi9Ohfu0WQp4ov}A^oxFNHCBPI0=|zQBdIhnBZ167##!tpy_3~C~CEh2c=Q=(xtXGfse!_{?esZ zb~GkRtiS^Y+~r_6C{18UcHqoQES6QVD+WZX^lB^d$fn~4@tp^WAbx~Pgh4WS?b^vRpW<=}(*!MIN=Xd!QS01L_ok9oB|NxL_EDZF^xj3c3Ls zUr?Zu&TUYicH0B2w?lqoQIBzDh97=^v>lXsmar01QMm~s1X{iEk+JEKGASEw8$P59 zqbZ|o3~5HXW&-q@*1Q&*A?qLZX@&(eR5!35;z=_`jKjnZLeeLM?0QpgG!guwH}y zN(n)=xfa3S3TlEt3)Pwz1h1++(7_5EsB|>Ye%?zdPCpliNxZM1uL>pcS0%+P1tUE4 zC`RxP4`iS~A(HN5C(Dgsb8ZsF2XA89JjM0Zy&s7sqNo!F1-xZmi(&%<1&`zBMNu(= zW+=7-l~~ghQ-MT<7A4Sq^%W>poaYP6_g#&H5hKOj4+=IElln38K0$~HK)KN$K?2@Y z0V|F|;YCEe_bfc$HWsW&>=@(1L8Mwl2LcBei;F%K!+xPXJ|W;OQVZ;6#h*cuCRQm6 zeQ2eu_Qo4iZ{+P|^VGurJJomnkBtVGZIW6w5w6M`Z%(~APu*f}GRy3qEW0Pi?pr+n z@%KOa{_=tT?1BE=f#aF7;~C5GZ&n!3GSir48XpoQ{an7Re7ZYRvwIJ8;i( zV1?O&hYl5oIu>^HzDXKJ~AokW+xl86GVm+>pi3PMiR1NnD82gyheqJauHrJ z#I7hN#7ki7UV~!d;@BN46gc!SY=)`eiG*rcL%`$;SZ(#~5Ipdv&?=Cy723^sbQ=_D zB2U@gIrY}5x&7};3*CQn_0DtI^6n+7C(k(2gO31$@UP_o8@kFN*a$v>$rQOcmzp8> z60ct(h?D^tWWyLKkk>7;A*Az<@fRLsK*kS?ZUz}K^`W9u(I9ZGVO1Rs*1HW4qBIF2 z@~Jo|aB;c|lGn_<(hC9v2Z4-92hdJlzf2J_sk#^4DS?|XlEek#dR%u1bn-+GScs$^ zW+FQdXBrR3n0k(4zul&uLYFSJ6s*q64#CL*_L1Q6F~za!2_GDTH@FO__M5b5un+Ji zBn_`cy(Tc}IGm6enW&fq^+hOyVK8GnpcrAkVtW}QE*LAVn2PR&#^u282RCO+$c)Frt z_1;Wf$snPr>!#y|(s;@slbb0s7{VqhtgWmmGPnZbN#`CI*b#zw4NNCxOi|d4;OndC zk{DoLePl|~&q9r6k-Wa;k5ENz%8xOi^IM|q`Y|vJqL;Wq`waoYZ+IQdDMiaPqQ*#A z!6MT$hU*MPq)aJV!YW#zhtx1gZoSW>^^Jhz^BYL`rRZ_OOD9jQH7b0o0xyU!W7>T> zAjA@Itqw-vLRPf4HvLw}wt>}&5*S*+0@@YY7XdG-SgS%@uw^ROq0@M?}Sjk8-#VM*T2=f_{Jv^HDfSV}b

S_w5l0d7gz%&l%Ap?r3z=?>JLN!Ir7~zD+##q8sFvAxB4z`{UKZYVr zd}S|tw+{@Fsa{@2pvrPL!^3L8Ib1dDPr>yUseCyb8f3k#3!0Ch18j&$y-s?lJZ6cP^k`^)Br=gw5FExM82NPoJNC zb>>wSzh~DMmffpnn6+vl>RSHOac;V6VNYiN*?YUs{)o#s&ZYaISYe!#iFZ4HHa0i% z(>F3qQ-*5#{X-++941AqnV&sl?V+j9ycCpvc-J4S!r1s8{SWxv*{57c5eXO3;#rl# zMzhKJQMmfuHhow$Y1OzkgEJ!wvfj2m6^?;xqJ0CtCm3~#ES)%;U2rwyPZ=bOCWXx| zvLW4=G$?|2$_(AqN;6~wa%c`glbmq$DR?SM^R?DfW^BH)bK}b5771Td)CZMGRa!fQ z?=u^o+b(T3e0jSA;QFL`4Fr{=ihu!C-u7rPufcAos1%(tN%-=kKHw;;WC~SdXQ|bK zN2Y}enGQkRVdJCi464TWiYSR{KBNjTDf2q4wNisNH?&=Am1Ag zZqe2x|FqrSineJQowBT>`kANf;U>E-TQp9tlEOBis7d2;jgIfs`bu?ao;Lr_SzD<7 zuB(9zS&(7FpDDxbXa{Q4aHOp3q}cQnPByVx+4>!&@Wi>m+fLxES)*&6et?E=d_uc7 z_7DVWc?$KL+p$-(GObS8*2!eoQ?PEPS+?&elPAs<46+SmvRji$ih&CczP4_7u970V zPG0(DsRS6>jCUJq)p@p~>XZZgkW)4bAIOYw>nU7swwrd1YhiMYH)})fvK@5

_sK5VB~+pPvI8C1 zR1!`N$A|%kN}?9@oLmNSg=-vo9`|T_0Ozes*tHJ(LBLv#zqn+FK%+ylL&ZR%&eEKy zd)ehAv^~i47QIJH5hfP=SMc+exs6Sxclme56QDCIQ8U>|&1KqApm9LY`0NRU= z%jN4O^MsWL|BRI@)~$R3R(?^g*s$_ZjcRoAhlFhG)xY0$8CJ@ba1x(-&yezfbv~=X zmug{q=~l2tqmPv!w3h5aZnXFIkaogpYmh|r0ABlbIE_TD|3r@>ygHOT_@`zNyrjZF09g%WLkvD?YHPzE9S|{qfE(|Vgan_g}knB$J(^9WX;AN*h}_I*ZJegnn+N@ zfn`nh`W%H8VqqLC3aGgpWox1Kts};Kct_PV({^; za;|e)NE$&v{09k$Ami*7d@AdyB2lqk(Go~WrkI~7o9? z0blRnIS8bxNi4-(sC|&2@$pw7f*>ZJ^T*;7MP+a2uX2%inD4;M7eG5e?1<$NEP^#o zw+6+kHEZ3ke|=qC62*ai|HaVi2%GhxCndVj-rNEH5pAb7&ABM9DfGu-GMnjTEAr z$=cEw&N-=h2bPm?<^kV8fUgMT1V9_16$`e4e^kuFFZJ~F4G$}2YW%`iO!tNPWOWnj zShK!901-{_Mqxm~C3lC#xIf75JqCG%v5sg0(ql5nf{(Lp&<$x+1lA?XEOv5ARUn># zM1vYA;*l>FEpoYk-70Yi(y&7>5B3grv1fV)fyuETBudzpU=%W_g*+Y?_9nj{7TFG7 z@^^?6kV%c8V%;H%qwOLu;7RT1k3pa{AnS=A_p@zbr5>>OfLvcG(dom9P9cc+W?qqd^2oTO9ln0Us-36Ar~L!ypc% zsVUl`?ApXgP{qiCcf|o|{%Rb;Y&^m$jx}ms;1tJC_QCHF{=j>YAk}P#8vcWnAR_R& z`4{9Y3Fjq45fkL!86jb2P%{%lA3$D1F|mBOOJv1E0G3r5?I zsA*~OSi3&(-vBOFLh--EH)-OYy=l%pd3I9z@he$-)9=-6$J=FYZ6RR`0OEr!i1XHx zV#P5z9}eEBbSTuBF3?4)ziaU1w!&CSk zrr!wa4`(_SF&OU^#ai+RR%|Nj1jVZBe_=y`FPu~t5TFooCaf4l-B=Ye92C|u;imvL zh~bAI3YaEV9G+!IW7g4_uN;Dmqtle`hm0f5EK}84s(Q|{uqAWkT!yM%qJ|-FR_M)J zJb71n-Z8jpCg_^S1Vx)xZTg(Nr959&pLYzb8?~1&Gh4FEmK?J!-TM{exN&Ie(7T7z zy(`dse(L!f$EJ?GxBVB*KX3j;>(5)~rG?jXb%&RjBmdg{>w!-PetqiGQ!5tR4fB+_ z5RIKXoU=3oxqYPRv5Bxc^Y*hV?#kJ^nY!Q6Mym}Xvo802M_Ic6D|h8|aK1j{X&v#F}pF!=KGGr8p?82rnB2-wq@A; zzqdehWKoEoxAC-o7nw_x!f3X4ObInid$y zuwVG-!kw|)f&NVMz&-oGeRs#=zI*$IGMz(N_i2!k)%LZEuzObQZb$~^T}^pc{Tw$_ z_S=eWfWqB0f9;-Y?|sjn-#N?BA|>_wf9}Ma|cwh;lCI(LK-qt)w4<)9ZoPAPt9Y-RArm8sZw+U zRfi7W;r`ydRN1j;T&moE_iSmM#=PB?w|C#YkhKpz8NK+Yp!Wj$RRif5Cht3|zore2 zn#Wd^OaTeJF^aNy8phJd1vD72@u3}bR#qsnyXqM7tb#_ zJHN0TzVEKanbHg7WEbq!!k&fmfBXE+syn7^!?C;M-JSnp{d~{oum0QNO!+Hc*k8rD z)C=TS7RO{@&M*~Tvef^^RX({VL)FfaP<->4GZ2+E%N5Ppie|v-@F*^ve{~e|um4f8 z13sd^sI7SES>qQSB$SH92Zus1>hr;c;8k$FLtdw1_Thhp_CE@;(ust%_$U_h745ag+%|W^!vNm$(!#dtT)~%kQ i3Ccd1yl1SSN=at?hOC{ literal 0 HcmV?d00001 diff --git a/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc b/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d355b33eae7a8c164d701bbfa5977143f4d34cf GIT binary patch literal 10719 zcmbU{TW}lKb-Ta6bU(qpxio6JO=WTwtzT6dBUfRsvP%@55qAJt4hs7GfU^{40D z#e<|MJMK!nd-p!iJ@>p1erL0B6g>a>@|n>On<(l(@uT=Kh03E@sNA3g>KrA|f-XXz zqsd!$PDkGQb9(Y-&N1X|IA~8b}^JB1HNQs<3&xe9h9wvoGV2sFzA_-X$B|bQW zrM68tI?SCvACCklF2rJEa_87y*;g3npBRTV{QOac4~>YSF`19X1QFL002o=`$_FEn zSSY9mJvjUz97+&}f{BR2#}Z0B zq40nS>>o*t2cuWO@$;`CUeBNR`9|ef6d(ocaB@fr#}z&FZ-F+P}3V&g$P3x`MRI?4x!;TmOf z$WSaQC&oo`6Mz|nSqMvbQ4?^K-pgS?GvHU=&u`-oA!>_Qj|XL06t?o)p&8E>llbu% zTy)`5hJxV;baOkPJ0Zh{I6#EUkYZ6nM4}`?OGkk!;1U&nD3*u{UO#sq3A8`y6o3+A zaoKmVK%B{{wJp9-LI{TR1;~vqIr0*9S$~$gM0?H2=aE^1aLeOB zoN%F%s3fE)kQv`0g|-5J2tdT-4OiwRga!?~6bvIR`Uky?Y7YhDFi8w(ArSl`pPWxca{$=i~RN+zkPwT-IsOrW!b(jmn>C_me!o5^&v$Yp36HbW)5dPTjqvyp50l;?s<0ilBFKI z_7%E1Z?)VCe%hY%?9Dp%&a-=$_0ac3_5fgSb$Pn#s9)999k%O#wYL(=U)yLXLnU5; zK`i_cPWl_dNdkSDp2R;L@CP-zawH7IN6-yZpA>kkhx+aZ6gA1DXhEN}L5)r^>&{`q zr8(>N8&<8k16DGm4C{_G78RRdLS-n^S3S=d$&^O2g^|9Lt^lWwrlK@M{Xd~KmrpZk zG*c*m{AOZX< zg_EelOUjlX78Rl&W!@L#3u;A(g+Ss*QBNarpC+ zPC2kbt%8iknS?iiyA^ROtqJ9GE5vQR7yV%4tC2NmK&?RW%SGRR)1n83aaUaC_B2c9Bsd zAhhsE)d-{m6rP5v09j4i3awuFlm8nEU|No-Nm0Io*PD z%R`DYxYA6XwavJuxwIkg_T1>Z-k15-thC_Xyy*7l-2S=L?b8eH7avl3b9K5W@2I?X zdHQn3xZv10yJx|%c~1Gq%O799%`S9yefH`?=kdI=D&xA&%^3dRtXej~oMo1BRo!5( zvl-9qjs<7?qH}Z3xq0r;?dApNp|oLHr#HLuR_719Glmb$@0qjBJ8pF^c=j&1_vNhn zrhArM0OE2rWp&N8W!zZ{oTlN+#|Fw-3j)II`h(R8!oci$B3HnHe_hwuP3e9^nY*oq z-&klUSBU{sibM=-ULgi1b$^x!xCHE~Um*hY>$1StiU7e_WDyW3pwNh|Yb>+u)!op_ zj_m>YOJ6+zN_Rcs^mV(K%Yp_)42X~OW#&aYJ7wshii*9+sxV4k0mp>KvO~H7%4l+fT%%>#*0NE8K7YbH?O zsIOHQki8NFqf#0mY$(dBZW0B^PWXHDJQT+%8qA{<_1!8WoauFOMitNb*YpjVrmog7 z)CB$adN3Pw(k57>-%Cr)@b>CdoqwB3%QA}p0-rrG9|SW~yd0OFPqvj=4%jq+C-omh zD?K7V?=KBI3X_R``Y-CsDQdp%(A*oDt`Bz&>7L4U69(4UAxeygql_s4xzJ+Qdf zsG14tP$)?4&`e;HkfS9(j$$>b1f~P(d5{C}XriFvX}F|)Fy;{a$^QVw6m`#Cm1U}? zdegmm=Y|`$>$c1=FuHRyih-JyV!-Ksnql|cI=tZA12V$moH;&k*_a28d}aC=tYvj$ z+tGPT16C_*ZtT6jH`6zJVWHBySn11E`m+A6&vqi`AN992P)u;f*)EoE|r3^2Gy;9Uq{SIG#O39I)N3; z0vD#kv|xFk8Kwm*^xNKNfUfQRs_x=;Revfbz_l~zWvFK2Ys7;gkn{Z_xPeIy4Rasy zm6Dt$;NdHr)cH=R#z=TPtjH1%AQ9^JG7^#`Xp(Vo6jW2e5LC@2R}FTu!G{u*lvA-t zI5a^tj7IA+7;#)M9)4AnWKcLNGZqHn&yI@KMyH20A_J`2sW2qhVFhGcMJ-kjOtnt+)EAZKMUmC zz4sbg!JcdI<{O&x{MI=kyK!f}d2`lx==K}gtzFrs!w<{`SH-g4RA+x+;9T4Tv%_j! zrW^*Nv>%pR<;0Y3Z4mAbZw*X%-9Z>tc4jp5f5i!0^y;x@!B8GMTrz;Z z1M5%fCmE1JP2pc_$?3lKwCvCl9i*gm2aRJi0BMH6z( zqgaX!DZ02y@qN=?g#H`DL4OTd}LN2yyu>)D-8!jeZmO=&R3$&{KwQ z*ic-*02@nS+dy<|AE642EOiUDDHh;;fzV6h0E7xPKWsKBhIV(HH`i4TDjZsw{>*z!8>{t)W-~ zgD?=*&;}F&M87qKDB91Yd0Wc+6(1g(Z zMCXeoJ*WobL1pC7_xzgP^AodS5_D!KZe~0}LNoV~y;UX>8IKYs?2Qld1%|U_{h{5uqbSgg(ijVMKHQm=fUQqkxw&ya>ResRHm} zP!DliP}$OMs3)5s0uU&Jx{fW8f?+Zb1+gU_Qo0T6K2=Tiih;<3#g~aeM7;0XN7A!acJF7;xdl3{Cqxm#KVmvMrO#_Y-gfFDyILIJJ zAQq#Uo*b;KgfJjatx-=r4^Rby7&B1F9Y)Aglmh7Y-$ckn|l`OGdgu3y3_L z8Dw9LPALvUf;jdUpb@92{Kl54BUwvbj;UW_Oz90XeVMVFFJup%%CfIyzy11MCa~mk z&xB_^a|eFu$X4|&xQ@PgZ0c~DPVdZ{%;{I&x|C<_>12+rU1XbbY*V(m@AI9x=KeeE zN$7yND!XY3OrZVO_upyUcB^WkanC~K-p2-gg|WB}I7BSB-b#l*8^|^H-C>X7O5kTr zOi#S~z07MlN9$By-fExfUE=K5ywl!yw`ACPj$f*&&D*Lo%=@vq#4YicXY)2!w$caQ zkgfBPUhmpO@_txOYFXx-v)5i3b9J_I; zb<+bvm$i)_biUWQP`d@CQsJmHToV&_e@(?R-BY33^#EQdW z%_B6X44^3us3}u=youE|UG#>)!~7=H&k#VbD^6D$inY(g4tN+S5n_y>C671=eaY(oN zlqt5bD-Y_cxqOadO?H3f${l5BU|bWTSsGh)OWtSPl3DOT7?qo}fOVyvyBVrp;!4&rtrs&c3B6Q`q!P-7@7J9UP&2adXn#P)XccCH-9Bk6 z&lI+&Y^!PQY6U~B=-+{7!c|v~Dz-{G!81S!J4q_2bmme1=`%qnxDh1WS8+{q8EQgz zMxZ_5$g|W8ef9Un05YVUzHT1|L(VW~S8D ztU^|n@6e2JOh^l0h@p88iDS%=5;F}<@zR`G;aH~@yy!%`aR2rl{$2j< zz6%hA^Cd1MqDo@N4*#}Y$$Bz}7)h>t^!{C3!BMI<0yzB~;Fu)>m z(&_U7nfOX9AcHFzN&3MV@gqvKAATVHL-Y%Ra)R%|1YUXv+>I|**vBiC@%nu{Gs)w& zmCm&~>RmdZc5@Rj0r?t&b&!iv%`)I8FeXmOUNgb>8dOT94c6TG6Z%k+J<@gbgmz=G zF}N`;s_0SKpfpVGO0{=M!zAf-3a?I5twBKuKrd;N&=Hebpint79D8I)W{o72a6~m+ z8VQF+Ao`BK!2lm@LY6R4qxoPt*)`29?-3tiXjU$Y71Ur;iWl;%s8YJq#r@=+wdp<6b$_-YROi$Xlu>c zTJ!F1@LKG~sUr~mG*}jynjBM;Vdv_z2Twyp`VKRYXE?1j&sOIhb$Q#s61U-jp0?L6 zn<#_lF=a3qm#w9_ILdh_Z`)GnWuTW^(Ys;E=FB@Q^R}L4^SaYFkWIQ*O}ADoTIzF_ z`UOklRL>tRu4@OU4`$4FEX_--Eo;J8I>y|#)U@qq?w{9wT6??uryCcVx^mV=a71@!TEP`jZ0OYhdSCeK;LuM zJT@`a;2?wV+&$xh>uPDuvMp(D=3L$aS<-j*K*G{iwd6hU%btHZ_Rq&YlYen?!FwX- zY@gvW$6({lyYMV+Ikr9T-C3;u!BUmkJe!=a-M$u*J-7FN&gBlA&JMnw-5w=QO%Kb70_N_Ozf{GKyQZ1v2=jPk*i_paPx7PlSFZ99C2?Y>vj zkQvR@c(aw>sUzuKZ=Sekvb;U<)&!8nRpHv`^k}Afj=nW`$G-nAdtjO2+}uOTV&0bS zeq2r2Dty#sJZJ9DZ zGII33hqW%c@nMIZ_B?EM(#D7FEj0J=1pNk0w=R$9DaM*k&g(bi8S7N{+kJ2LO%J~N O>Rkp9MKQGmlK%r-c2V*G literal 0 HcmV?d00001 diff --git a/scripts/validation/__pycache__/validate_slurm.cpython-312.pyc b/scripts/validation/__pycache__/validate_slurm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca6b456e48dbb7c2a0a2c0d6f6df761810a1d93f GIT binary patch literal 8561 zcma)BTW}NCmhF~WvYxj5!Uh{?8{0@A851CYV<5Z?B!ncG+*=_evQW2e30YFQTNo>H z;^d|(GNB4{Gc!1oI~6B0Rq?GbRpjfEuSuq+YNl#FMzW!-Y5b9@$;a^L03W&h%&dK+ zmLG8*N2SxJ&tso`_Ur7u`#;@o2Z!?c3$H}28#wNN=*1{PsqyF<8aFwKJI_hHWQy_U zdG<7&H?gPryqP_P^8$NX&Rf{idfqD8B=h_H`3gyRpF3}pEU4|06}3ZlN)?y99H%sz zIN2I)G&Ws&qTlgl>Gey#*xv=~ zV8V&GrX*r9S@Dap#9$=u7ZZvYP7Ehg*fgGyu=9W>E2aG+@j*v_e=-&ty_iT`Qg^6? zOCTIeshHC_Ivnfo?-WmIVt7anUs3_bfDQzKSmBy3_G5}a7x97G(nDEjwp#Z6BAic(Bn1uJ`Bo8DmbBu1Bp~z@?rNg2~F-M z@mMKqCH|^@&I4u?{zOmYDRD6(THB1tkprZhDIfn+BL0DT-%A`wtK z#dAYo511}rNrIHJ^bDgf7%`MkA{uycnZa`~#?wS1e}BIUk-|9u10CosF`opj)A4}x z@l4oM(ocF=@i*iDNQAQ9<&0liF<|cFws2Z`LDW@lmK$xH;^aBWED4e&!>57MfH`b> zmFwfQN@Eh{RAP=5b1HB`+jLY=v>PuTM$$7Wft}4VUpG|kDsf$S^0RdEXGnlSBiU^Vd>eeE5x<%Ds5!fq` z)GN;wcRqe43AL5<>R#v_6sMPz=s4*ZkT!>hC50HO3j|*!nGr?tNn26C)E$?Lb{(2x zH02;Ork%-A#fjOh+K7hwM^xk7qF}S^U#MDt{q2dj3$ChZVP^N8hR0@;$B`8lt(>Rs z`a2WvObcH)KXcBDJM-esIdRwRbANg5tJmiD^yc^U&h0swt2&vpo&0gZSwHV=%RAd1 za=hi~LRHP=kzC{UnZbPHo?O+QS=*ikXEP1$FAeRuz2$c3tB!o*zFgJ5S=+uvGsd2% zbep|*xZ&^y?%NGbN8ILb_tv8RR~L^uT-FLqvf$5jW(vI@*l%(X?y7xBZwz@J<1<{8 z=B+Bm_%ZJFcR7yf4?$d2j9Fu5t-=_}n2jq{;xFw5N%?E78S@43eoPp%WP}WVnfm-0 zel4xZ2*zrj%UGhslU2poGwNP7fLF<6td3UwbktZmW|^=@tA9G4u}J1GOZG$((wHz| z&sci!y!HoylPqIat=3qTwGZ%cH*6W}v9fJ2_8zNPMh(8)Ys=U&74%QTf=t*4gg(w! zk-q;@NP*RbQH;a~5@KLT44e@I32|$CTx>tL6=t%ifw1~og5BT&Tz~Kq!jOCo_R;|f z7tb;cS4A)R4r)+KXrWl~zE^{x%ebTH!ZbQk@nu~Mmgu8&jD)zvof!9;qimq7E z8%yB22!~X_(O@cYs}G7>w{687C~8ucd?FAes{_0`8Keh9%(P?M_P4jRw6{I`Mu1s> z2PCF{7IYyZ#bn(w9JvC0w&Ti8gmErOchJohlQqEqiA3+h(nm*7{Vo4I$Kx)@aIdd} zWbqTG=yK`D3@RC3;tY=d9eHz48y$8Mot~upkx;;r83o?EMi*q}B2>F@mpgwSh#dU|Cy=53<||Ah*NU377kp7GuVXXS@`#*Y;|>*hTzc~8rn zXUjv*ZrPa?3ZA-~jvJ1TTv-byWm6w^PgP$%RB+W5)-_EyvqDxbIIAXiPxMfi&2`;2 zVVkVGR$1`W-*nz^e&n9_wBflvj*lJFduCeZ8vQeG z<{P`R)@KIwNuqW)mBux zNs)EiKqM|j5J~A~*jU{%s3cNJpHWYb)| zH}CRJ9RI)#Im?JwPywf4R`z%4GZvobu+c_}dbB+9jTk!6L6I(ag8K zW&M(%qTqI4dmaLM$@UxO*)h-Y8|FDBm*l=x`K$X@X~a)fB#%^mtHy{O7^Z9JRN0&0 ze?^yS4ce&9OS&V~X&z(0XpPZYdfuuxG!Kc1by5RP(I7R_FT$a2ptnIzTK^~4m`STQ z)=9O}hQTr~E`Zi!X6AL7rN&}PBqJ#6GD4KpW>qoHBQ+K6iPTK;y967%ZT!6K&6HtV zGX$Ibyv-Stw29V6sajQ}%^B+zUbP^C5i@3rTj+04hUem0sRip>*`7_t;w5kJ0&dE& ziZR=mearzb$|P-}nee7eQX6a8(Q0Qc2U;DhRZ%*19eREqv#m>5WbDc@_@&Lp{!34F z-!H=OX6%wL3JbZqXmpXXB5ix0%Q$}OEgHKIm{w9`1?9J|L3sFv`kdI-fj~FjIX?IA?j?eiqCt!4} z0)SVgI^&dnW^B;J?hR+g2^d|U^ToF%s$^&#PQSX`+G%LcDj6k8wL$({8Fo(*b{g4u zR-hAP;8ztxCk$HPoV#dUycMmVm#E!L*6pC2@Jljpz;NOAmi91S@lm>ARu!dn$!=&n zjKOG^(OP-}&N`{(*3*WyGw_Cf9hLUt4u_=}b1gy3xIW<}NOs0Wa`6fO59PZXo^k{3 z>e4B(64q+Wy|hx=pK-60*K0Kbw`{zd&O}gG&N6xxlhoM9NzcF+=sA;a`wa=#PB`jL z1UD>EhzLGyEhZ4*Giyl51vMolr|aZlWb!Caw=8MsGwIe)nDUvrFeJy4>B>Mry%b3X za7zb}gi5#28_P{mW@>q+6LEk>d8+jKK!75Yk-)OxDZqlQ^nQurGUW3R6Hyoi013oP zY>*OZqLdgxR#igG6sJrg(t)Y?r8qj%&Om_t^9bRlkYAbJUw%hkPGLAv#?`<>^p-NN z-C`*{Xry3$IAuj?P&rJXssqLW1En-g#E)A#y-Ssm4Mh?X;o3+DJ_4s^_UXjViq{UX1x!MwVDj zB18#bmW@qsXh;4VH1#5YUv_2&+vFElge~NFvGAE9wQ}(6gjyq2Dsh*2mA_`o@TD-> zy(~dXRqb2Rn+n&0xEszQ_fNK3aTOvPbe=?+XsTM47Hd1>>Xgz(&r0M zhGZ>1%4njT!F-Zi(&@B~DT8`oC$&f&4(#!n(~hOQdU|sp@McO@MoDQ7P{zYADacpgtxbWwZ~mmoeY2J)@fZ6Rflg&=z6 zVPy~>JL1|)l&Mg37qSOIjIusuKOpK3Hj_Fj;W3obA~D@EG8739A!R@xVUSc9RTYZ$ ziiuGQoM1+g=!r`#;MW_NK@2XFyCBm(U5F+kaX`iKWQj#Vx{E9}4aTlR`56 zE19TbY)0+|z~J+cZjn;MN!3@WTN%cRE);Pte#Ail7uL-r<4s&m{p8@(Te zzRu)ZPv5h>@SQJ^wdShY^R|u!Z}**DbKXOFTYJ`$O#y|=Ko~oWB z!_=`)Pknsqp9Wy1?pW{ougrB^nQI=+)ua(VSHHzSHk;j*j~$@NZ2kVZ>h4waSZLdI z`{-QTzA|zZAb#6yMf*%gZs(Ewwxi#?IlJxTYC61d*LLPRXB8MY>ujE0m-8N+Z8`K! zZ|?aE`D3r=`a`qFLIts{bTqxPGiyek?E1m02k~uU@`dT$UmX1GV6OJ5yz8m#(Spr6 zTlH$*_G;F=(6C`@*Nial?a6z4?lv5laAi%|qm#7-r+c#c>fRe_VawKR?_^KjB`)}P z+&2HE@~g_Zu4m``hqJx8TJNmO_uUHm6+FI~zS+Qu+ecB|+564euY2;Ilh6rw<)V#i z+%y%-ZJ%4O2nMA@ zI2cr3$GZ}v3c)O=gs7qfs3K8?dy->jkJ&S38VDNQ6TzS^$oPP){GR4eDuo%5C$^s9 zgp`WO2Nn7z2G5IA>0$%IJpY4G%?pn@INo!gbKd9d_c`}{uKGUbcw}|(`yXzo=l4GB ua`M#=n}>P6ZL#0X39f8<*1WzTxW 0: + hosts += 1 + except ValueError: + continue + return hosts + + +def count_inventory_hosts(inventory_json): + """Count hosts and detect DeepOps groups in ``ansible-inventory --list`` output.""" + hosts = set() + meta = inventory_json.get("_meta", {}).get("hostvars", {}) + hosts.update(meta.keys()) + for group, data in inventory_json.items(): + if group == "_meta" or not isinstance(data, dict): + continue + hosts.update(data.get("hosts", [])) + groups = [g for g in inventory_json if g not in ("_meta", "all", "ungrouped")] + return len(hosts), sorted(groups) + + +def main(): + parser = argparse.ArgumentParser( + description="Preflight checks for a DeepOps provisioning environment." + ) + parser.add_argument("--json", action="store_true", help="emit one JSON object") + parser.add_argument( + "--inventory", + default="", + help="inventory path (default: config/inventory via ansible.cfg)", + ) + parser.add_argument( + "--remote", + action="store_true", + help="also check host reachability and GPU visibility over SSH", + ) + args = parser.parse_args() + + root = os.getcwd() + if not os.path.exists(os.path.join(root, "ansible.cfg")) or not os.path.isdir( + os.path.join(root, "playbooks") + ): + print( + "error: run from the DeepOps repository root (ansible.cfg not found)", + file=sys.stderr, + ) + return 2 + + checks = [] + + rc, out, _ = run(["ansible", "--version"], timeout=60) + ansible_ok = check( + checks, + "ansible_installed", + rc == 0, + out.splitlines()[0] if rc == 0 and out else "install Ansible via ./scripts/setup.sh", + ) + check( + checks, + "ansible_playbook_installed", + shutil.which("ansible-playbook") is not None, + "ansible-playbook on PATH" if shutil.which("ansible-playbook") else "missing ansible-playbook", + ) + + galaxy_marker = os.path.join(root, "roles", "galaxy") + check( + checks, + "galaxy_dependencies_installed", + os.path.isdir(galaxy_marker) and bool(os.listdir(galaxy_marker)), + "roles/galaxy populated" + if os.path.isdir(galaxy_marker) and os.listdir(galaxy_marker) + else "run ./scripts/setup.sh to install Ansible Galaxy requirements", + ) + + kubespray_marker = os.path.join(root, "submodules", "kubespray", "cluster.yml") + check( + checks, + "kubespray_submodule_initialized", + os.path.exists(kubespray_marker), + "submodules/kubespray present" + if os.path.exists(kubespray_marker) + else "run: git submodule update --init --recursive", + ) + + config_dir = os.environ.get("DEEPOPS_CONFIG_DIR", os.path.join(root, "config")) + config_ok = check( + checks, + "config_dir_exists", + os.path.isdir(config_dir), + config_dir + if os.path.isdir(config_dir) + else "copy config.example/ to config/ and edit the inventory", + ) + + inventory = args.inventory or os.path.join(config_dir, "inventory") + hosts_total = 0 + groups = [] + if ansible_ok and config_ok and os.path.exists(inventory): + rc, out, err = run( + ["ansible-inventory", "-i", inventory, "--list"], timeout=120 + ) + parsed_ok = False + if rc == 0: + try: + hosts_total, groups = count_inventory_hosts(json.loads(out)) + parsed_ok = True + except json.JSONDecodeError: + pass + check( + checks, + "inventory_parses", + parsed_ok, + "%d host(s), groups: %s" % (hosts_total, ", ".join(groups)) + if parsed_ok + else "ansible-inventory failed: %s" % (err or "unparseable output"), + ) + if parsed_ok: + check( + checks, + "inventory_has_hosts", + hosts_total > 0, + "%d host(s) defined" % hosts_total + if hosts_total + else "inventory defines no hosts", + ) + else: + check( + checks, + "inventory_parses", + False, + "inventory not found at %s" % inventory, + ) + + if args.remote and hosts_total > 0: + rc, out, err = run( + ["ansible", "all", "-i", inventory, "-m", "ping", "-o"], timeout=300 + ) + reachable = out.count("SUCCESS") + check( + checks, + "hosts_reachable", + rc == 0, + "%d/%d host(s) reachable" % (reachable, hosts_total), + ) + + rc, out, _ = run( + [ + "ansible", "all", "-i", inventory, "-m", "shell", "-o", + "-a", "lspci 2>/dev/null | grep -ci nvidia || true", + ], + timeout=300, + ) + gpu_hosts = count_positive_stdout_hosts(out) if rc == 0 else 0 + check( + checks, + "gpus_detected_on_hosts", + True, + "%d host(s) report NVIDIA PCI devices (informational)" % gpu_hosts, + ) + + rc, out, _ = run( + [ + "ansible", "all", "-i", inventory, "-m", "shell", "-o", + "-a", "ls /etc/systemd/system/ssh.service.d /etc/systemd/system/sshd.service.d 2>/dev/null | wc -l", + ], + timeout=300, + ) + overrides = count_positive_stdout_hosts(out) if rc == 0 else 0 + check( + checks, + "ssh_gpu_visibility_override", + True, + "%d host(s) have sshd systemd overrides; on Slurm nodes GPUs are hidden " + "outside srun jobs and direct nvidia-smi over SSH is expected to fail" + % overrides, + ) + + ok = all(c["ok"] for c in checks) + result = {"ok": ok, "checks": checks} + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for c in checks: + print("%s %s: %s" % ("PASS" if c["ok"] else "FAIL", c["name"], c["detail"])) + print("ok=%s" % ok) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc b/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9afc1c83eca63a2703afbf195a4c1abfd4ab85a0 GIT binary patch literal 6810 zcmc&(T}&L;6`tQ^SeC_$&Cl8~S)17G5-+g0!I%)IjvfEhF?Eeor6kI9*cq0=yEE%M zv&5{RqEf0Dsj8_|DMYdq9;yn<=t!oTO?y z(w@2Z&bjAa?)mO_&b|DjzTV5g6CHYg;?X{a`4?7-Pfek)`B#=EicjTx#H;e3G9z^yqt&TG z@#~0>*2Pl2p4L63x}Re%GHT7Ij4Em3M4M}SyL`aAwUK>PvlcYHFVPH8%{tKZy+pH7 z+ppH&3^NRAgn5Y*P^x?$6dG7&iksph{_JP(8@e5`HOsa_V!uO1 z;zUo|p|nCQjaZ?uWh;iN5H*z4M~OnFLZ*?NigbCs*E2@w6Wtyk2&Jd&anm>%il#yt zu#Kies&2*2NsScjS>5F!y6M>7CpfX8c)=ZXmJ&!@u1$f$$)=4cCb&|uRG1J;#ZqCK zu|37g1X`rFlqFhjfzuFOSYNQcH^~2Bl5X3C=UEP7ZXXehV)y494v31G!$I-iqxy0S%T=L$V zOTtp-zHi0<#wO4CJhSJwJdD5P?#FW4?gt0z0B6W4Hy6+tETU0KCFOFcvsL zWS$+i<#A0(+T&B%&Po?_y-@csn6j>bm}S=e19$lE!hLJu%lYu-uiIC`LwV`g9q~?P zKCv(f;9Qeh^HS@=vBjpPnq08wVdSwiSZ;1!^nB5_6j+sBCrAd>z5aLj0-^PP;q_uT zhY^d9%1?`Be*-eA=?L3(TudC>g$dRC#ON$wLM4iBP(-+3cm;-g;vxdooSK3{M4+f3 z7+1-wARNiX!exVmpt{S$G_LLRFdCF0hRI_|C1GV_KfuvDyR_p#!ywS#{XfayckAcs z=f@Vt9!tl-`?RJnHm^$I-F}B1>+bg7Lg6R+ouA=fu!p(r`y4#Q4`rJxJXP4p1bspf z>)?bVSU!pZ7sFv?3#VD$Sg-HiEUKnv(wZg1eztYYEM!avl1e%|_(MGKqTa}@RyPQA zMi`tC>{ts99?w-As9?Cd~g1RTEg?3%BzAZ-s2Li>QQTSR+a8?AhEsHZP(#kgl!4< z{fM*$#&Oe@7tvTDY4Mgr6OdWmkR$50ma-h00AwsjysjxKMvHKcU{9qr@(#@Z zx!;)7RbAM@R7zHubhvqoGMdyJ@q&`HvDm8-*F{( z{Jr&#$hMCS_09ebzM;W;hu;b@ZJld}Pv;MxUT$AGd?qg)qQ=ovk2L&Y9B47Xx0GJX~1cc7E7D=tuSqdXb^{YT#gN=thDJLDLODg_s^!EX@f3IBZN?$S|a(8B4P+5X=gk+Tlz*u4%v= zPR%t90AwU09O*}c2$v#A8q3ZF0LW=mwOpJO!s}J+(2fEJt_ZL$m+iKYJ-#rWs`AUV>x-rct>ESb6JE_k@##9Y^zT=ea2@bEE>2Ny0bewaHH%kAr34+Np|o7|!9 z+`gWT!wm?X0ENzLOzUfFEq(cxzK7OI%bRmPM9vP}EVL~_`qQ%;G|1-^Ge3N4FNdQieKyn6iR)EcTK?SMq>P&T05R#NGt)~%=DG&^7` z34#ZG$`URREcX;s3$fYEkT@JwJV`yJ+ZMS1ij*hH-cOk~HT1fiQVe|zc-HkkUtkIN zLOz0C#Y47xVhP*y#ImceOtf1W_+-hZp$(Zvi2Wb^GEakXs+w* zivJymcYy=*zy9-~g^%)q&ZSsB5M8>N4-72#=L6^Oc*_24&V_pN!Pvvts`Li+=upH% z25<%5L@|is8j254$SB@Iu^$B<%*fkCfxE`SCX98B;=r@;0Wr3Df*1Q-ZB)b=m!495 z!^l@XzD@BF!3p5yb z7E$33BpL`<6j4rTw&L&*>PP~bo_pOM7{uIAcatiTW_USGaWiAsvTjVm)s9J~WY-`~ zRZqBjN8m*$k&R;yPlA}4v{k`n2Pvkj8q1#8D2mxw#7EUWNv?K{qSpx_TpIXmFW7Rql-5dFXRrs{Rk#C* zgnGS3@16=hUbvjy%YtFrQ6*|*QllF27HNz6l{)^AyZUL)jg`Jg1z?2hTk^w-UVc^^sNSy#f)5 zl|*EK5poMf1qUhEh5`{~)q_9|sufRcHhaXYh5Q=7V~U@GL_n4!O-_xBSAme>`%Cw; zU3fi91kfb7jUC>avK*;+GfpsZgbNOx&nfM~{rTK-qH2PKIe&Q?hzPUWU(sgmA?b3?>@@(BRkB4o1 z+Q>9E=bDeN1UhE@-!$x>A6jW>pY?qcXeu^td4lX&cK-HSOE}*W-eRz})!)VT&3~}g zbUfd5e2am~)+pP^US$`gwbn?!HL}HEZKIPBe0Q?{ 0: + run_cuda_smoke(result, args.cuda_image, args.smoke_timeout) + + result["ok"] = not result["failures"] + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for key in ( + "api_reachable", + "nodes_total", + "nodes_ready", + "gpus_allocatable", + "gpu_stack_pods_total", + "gpu_stack_pods_ready", + "cuda_smoke_ran", + "cuda_smoke_ok", + "cuda_smoke_gpus", + ): + print("%s=%s" % (key, result[key])) + for failure in result["failures"]: + print("FAIL: %s" % failure) + print("ok=%s" % result["ok"]) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/validate_slurm.py b/scripts/validation/validate_slurm.py new file mode 100755 index 000000000..c7006e888 --- /dev/null +++ b/scripts/validation/validate_slurm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Validate a DeepOps Slurm deployment and report a machine-readable verdict. + +Run this on a Slurm controller, login, or compute node after deploying +``playbooks/slurm-cluster.yml``. It checks controller reachability, node +health, GPU (GRES) configuration, and optionally runs a single-GPU job. + +The default output is one human-readable line per check. With ``--json`` the +script prints a single flat JSON object with stable field names so automation +and AI agents can consume the result directly. + +Exit codes: 0 = all checks passed, 1 = one or more checks failed, +2 = usage or environment error (Slurm commands not found). + +Note: on DeepOps Slurm clusters, ``nvidia-smi`` in an ordinary SSH session on +a login/compute node may report "No devices were found" because GPUs are +hidden outside Slurm-managed jobs. That is expected; the authoritative GPU +check is the ``srun`` job this script runs. +""" + +import argparse +import json +import re +import shutil +import subprocess +import sys + + +def run(cmd, timeout=60): + """Run a command, returning (rc, stdout, stderr) without raising.""" + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + except subprocess.TimeoutExpired: + return 124, "", "timeout after %ss" % timeout + except FileNotFoundError: + return 127, "", "command not found: %s" % cmd[0] + + +def parse_sinfo_states(output): + """Parse ``sinfo -h -N -o '%n %T'`` output into node-state counts. + + Returns (nodes_total, nodes_available, nodes_unavailable, states) where + states maps state name (lowercase, trailing '*' stripped) to a count. + """ + states = {} + seen = set() + for line in output.splitlines(): + parts = line.split() + if len(parts) != 2: + continue + node, state = parts + if node in seen: + continue + seen.add(node) + state = state.strip().rstrip("*+~#!%$@^-").lower() + states[state] = states.get(state, 0) + 1 + available_states = ("idle", "mixed", "allocated", "completing") + available = sum(states.get(s, 0) for s in available_states) + total = len(seen) + return total, available, total - available, states + + +def parse_gres_gpus(output): + """Parse ``sinfo -h -N -o '%n %G'`` output into a total GPU count.""" + total = 0 + seen = set() + for line in output.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2 or parts[0] in seen: + continue + seen.add(parts[0]) + for match in re.finditer(r"gpu(?::[^:,(\s]+)?:(\d+)", parts[1]): + total += int(match.group(1)) + return total + + +def main(): + parser = argparse.ArgumentParser( + description="Validate a DeepOps Slurm deployment." + ) + parser.add_argument("--json", action="store_true", help="emit one JSON object") + parser.add_argument( + "--skip-gpu-job", + action="store_true", + help="skip the srun single-GPU test job", + ) + parser.add_argument( + "--allow-unavailable-nodes", + action="store_true", + help="do not fail when some nodes are down, drained, or unknown", + ) + parser.add_argument( + "--partition", + default="", + help="partition for the GPU test job (default: cluster default)", + ) + parser.add_argument( + "--gpu-job-timeout", + type=int, + default=300, + help="seconds to wait for the GPU test job (default: 300)", + ) + args = parser.parse_args() + + if not shutil.which("sinfo"): + print("error: sinfo not found; run this on a Slurm node", file=sys.stderr) + return 2 + + result = { + "ok": False, + "slurm_version": "", + "controller_reachable": False, + "nodes_total": 0, + "nodes_available": 0, + "nodes_unavailable": 0, + "node_states": {}, + "gpus_configured": 0, + "gpu_job_ran": False, + "gpu_job_ok": False, + "gpus_visible_in_job": 0, + "failures": [], + } + + rc, out, _ = run(["sinfo", "--version"]) + if rc == 0 and out: + result["slurm_version"] = out.split()[-1] + + rc, out, err = run(["sinfo", "-h", "-N", "-o", "%n %T"]) + if rc != 0: + result["failures"].append("sinfo failed: %s" % (err or "rc=%s" % rc)) + else: + result["controller_reachable"] = True + total, avail, unavail, states = parse_sinfo_states(out) + result["nodes_total"] = total + result["nodes_available"] = avail + result["nodes_unavailable"] = unavail + result["node_states"] = states + if total == 0: + result["failures"].append("no nodes are defined in Slurm") + if unavail and not args.allow_unavailable_nodes: + result["failures"].append( + "%d node(s) unavailable: %s" + % (unavail, ", ".join(sorted(k for k in states if k not in ("idle", "mixed", "allocated", "completing")))) + ) + + rc, out, _ = run(["sinfo", "-h", "-N", "-o", "%n %G"]) + if rc == 0: + result["gpus_configured"] = parse_gres_gpus(out) + if result["controller_reachable"] and result["gpus_configured"] == 0: + result["failures"].append("no GPUs (gres/gpu) configured on any node") + + if not args.skip_gpu_job and result["controller_reachable"] and result["gpus_configured"] > 0: + result["gpu_job_ran"] = True + cmd = ["srun", "--gpus=1", "--time=5"] + if args.partition: + cmd += ["--partition", args.partition] + cmd += ["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"] + rc, out, err = run(cmd, timeout=args.gpu_job_timeout) + if rc == 0 and out: + result["gpu_job_ok"] = True + result["gpus_visible_in_job"] = len(out.splitlines()) + else: + result["failures"].append( + "srun GPU job failed: %s" % (err or out or "rc=%s" % rc) + ) + + result["ok"] = not result["failures"] + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for key in ( + "slurm_version", + "controller_reachable", + "nodes_total", + "nodes_available", + "nodes_unavailable", + "gpus_configured", + "gpu_job_ran", + "gpu_job_ok", + ): + print("%s=%s" % (key, result[key])) + for failure in result["failures"]: + print("FAIL: %s" % failure) + print("ok=%s" % result["ok"]) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 3b342498595a03a0a1ea32fec9551bb844f63887 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 13 Jul 2026 15:27:25 -0600 Subject: [PATCH 2/3] Address code scanning feedback on validation tools Document the intentional non-fatal parse paths flagged as empty except blocks (malformed allocatable GPU values, best-effort GPU pod summary, inventory JSON decode) and remove an unused test import. --- .../deepops_doctor.cpython-312.pyc | Bin 10441 -> 10449 bytes .../__pycache__/validate_k8s.cpython-312.pyc | Bin 10719 -> 10738 bytes scripts/validation/deepops_doctor.py | 4 +++- .../test_validation_scripts.cpython-312.pyc | Bin 6810 -> 6794 bytes .../tests/test_validation_scripts.py | 1 - scripts/validation/validate_k8s.py | 8 ++++++-- 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc b/scripts/validation/__pycache__/deepops_doctor.cpython-312.pyc index 46fc4005a73d21db723134f2aeb3619e0c6b5ba3..b92ba6ba1017386ba5493b394f8dc02afa587450 100644 GIT binary patch delta 168 zcmV;Z09XIXQPEKi%MA?*00000D_B)(q_GVN4gu=37Y^784F&-KWHEgZSpbj$jJvb= z7$pY*=d&IlS^@!5vw0%v0wD$sBHSiByhOiBxk|uuyHC?5OwkY{kPDbr*c>2`3G@oH zgC#BkGi3q*6d2bNC(#op@&E%3BJc(P9U_wn@CO48BKmU@0002a6C+g_0M!{4RWbwB WF#%OFli(-724?~R8t_z;rYIv;5j0Ey delta 136 zcmcZ@cruXpG%qg~0}zP%g=R5toA`;hy)h`OGe_~+d6}=@O@_`vh|0)&& n8huV!IaHG2s-$qJ0rORT#!!RFFVqjPr7|%}eF&JmPD2?0&`~9U diff --git a/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc b/scripts/validation/__pycache__/validate_k8s.cpython-312.pyc index 1d355b33eae7a8c164d701bbfa5977143f4d34cf..7eb449bd7719b99af2fdeea98e135c9d310ea01d 100644 GIT binary patch delta 309 zcmV-50m}a0Q}R;|%MA?*00000BUn{x=_0WX$q51Olh_F+0}KTflK~2VM+FHL*bESq zLzo4h37HAdA~LW;(-=6@6gklhIf+E@5ex+t@EQON5s)RA6wnJD&;%X$^a%hE8u9=G z2^H}M01XxJ2LlNev&IUL0s%{tR}F~)KeOQt83F-Nv-=OH1p$7uqZL2_0b{e|7Ty5` zMg{-?my@akAG78d4F(Yi0RUtecmaJ1eG6FtkO7Q1v-}?V1qO)#0048mv-~7B0vrVm zEZiwOyhOiBxk|uuyHC?8OwkY~kP4Ve@Cvg|ClUbxdy`=(MHB-KEbs;ZD=g3uE07AC zTJQ%04J?zfC?E+JOdtcy9|24tliDc98xsK@@+=Mk81f0J5s)RA6wnJD&;%X$^aTJA8u9=E z2@%;05by&52@$i%3XlQ;Nt0O(i2*&c;|&=C0Zz0152pnIe6yt$Kmh?^v*s4w0R=t= z005VhsskUh=ok$K2?PNESpbj$j5M2@^~l1I!x%OdON#D90NS0Uq)w4gnbQA`k%v V@+Knz3i3<=aRC7&@Mx3IDI@z_P2~Up diff --git a/scripts/validation/deepops_doctor.py b/scripts/validation/deepops_doctor.py index 2b9fc61f5..98d5ce5d3 100755 --- a/scripts/validation/deepops_doctor.py +++ b/scripts/validation/deepops_doctor.py @@ -156,7 +156,9 @@ def main(): hosts_total, groups = count_inventory_hosts(json.loads(out)) parsed_ok = True except json.JSONDecodeError: - pass + # Leave parsed_ok False; the inventory_parses check below + # reports the failure with the ansible-inventory context. + parsed_ok = False check( checks, "inventory_parses", diff --git a/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc b/scripts/validation/tests/__pycache__/test_validation_scripts.cpython-312.pyc index 9afc1c83eca63a2703afbf195a4c1abfd4ab85a0..7215da80ffc1f1db80916e16e27772a67796d0f3 100644 GIT binary patch delta 368 zcmbPb+GWannwOW00SJsDLbJ9`@5r_9H}f>GL!WfrTO{6 zVw^xRE~uCyCpVDC15+U=m@1UQo2|jXP$UF2k0ncO^E$>GOpLsn+gZAq7-cpKv%hC# zRNBnU*~7@Fxp@LtC?liZ=4aeujEqK;|MIjlN^frFm1APG*u02eg^AH;^Kn5*CdQ=6 z4~2Uf%{S+Z=rA#6Zk{9dijlEoa=Sz}W5(pq5?YKRljS5;Cc8^E%2xwD2l8O?ULf&- anURt4BNH zL6#sYAH+-LT+Ij(MW>^U_L9H&kN?W!TEel4DJkEDeNr_ zDIBS+S#pyD7!^4=fjllKPnus4q$GtKD8>U56BJ4nPT|egU|=W`0vgDgrLlP#;|(T8 zzRmS4-As(Ko4MKFGcqb~{>{4BO~KSCT2#K&s+?QVz(JMZ!>UBz9qQ<0I&B@C;$Ke diff --git a/scripts/validation/tests/test_validation_scripts.py b/scripts/validation/tests/test_validation_scripts.py index 7293992c9..3010a5bfe 100644 --- a/scripts/validation/tests/test_validation_scripts.py +++ b/scripts/validation/tests/test_validation_scripts.py @@ -5,7 +5,6 @@ import importlib.util import os -import sys import unittest SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/scripts/validation/validate_k8s.py b/scripts/validation/validate_k8s.py index 489e01382..98808b44f 100755 --- a/scripts/validation/validate_k8s.py +++ b/scripts/validation/validate_k8s.py @@ -54,7 +54,9 @@ def summarize_nodes(nodes_json): try: gpus += int(alloc.get("nvidia.com/gpu", "0")) except ValueError: - pass + # A malformed allocatable value counts as zero GPUs; the + # gpus_allocatable check will then fail loudly for this cluster. + continue return total, ready, gpus @@ -214,7 +216,9 @@ def main(): try: pods_total, pods_ready = summarize_gpu_pods(json.loads(out)) except json.JSONDecodeError: - pass + # Best-effort check: unparseable pod output leaves the counts + # at zero; the authoritative GPU signal is gpus_allocatable. + pods_total = pods_ready = 0 result["gpu_stack_pods_total"] = pods_total result["gpu_stack_pods_ready"] = pods_ready if pods_total and pods_ready < pods_total: From dc0f04a7d068cc9e9528c1b1af16548a8416cd45 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 13 Jul 2026 15:50:44 -0600 Subject: [PATCH 3/3] Detect Slurm login GPU hiding via the DeviceAllow property Live validation showed the GPU-hide mechanism is a runtime 'systemctl set-property sshd.service DeviceAllow' drop-in under /etc/systemd/system.control/, not a unit override under /etc/systemd/system/, so the directory listing check never matched. Query the DeviceAllow property directly, which is accurate regardless of drop-in location. --- scripts/validation/deepops_doctor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/validation/deepops_doctor.py b/scripts/validation/deepops_doctor.py index 98d5ce5d3..124058a12 100755 --- a/scripts/validation/deepops_doctor.py +++ b/scripts/validation/deepops_doctor.py @@ -214,7 +214,7 @@ def main(): rc, out, _ = run( [ "ansible", "all", "-i", inventory, "-m", "shell", "-o", - "-a", "ls /etc/systemd/system/ssh.service.d /etc/systemd/system/sshd.service.d 2>/dev/null | wc -l", + "-a", "systemctl show ssh.service sshd.service -p DeviceAllow 2>/dev/null | grep -ci nvidiactl || true", ], timeout=300, ) @@ -223,9 +223,9 @@ def main(): checks, "ssh_gpu_visibility_override", True, - "%d host(s) have sshd systemd overrides; on Slurm nodes GPUs are hidden " - "outside srun jobs and direct nvidia-smi over SSH is expected to fail" - % overrides, + "%d host(s) restrict GPU device access for SSH sessions (Slurm login " + "GPU hiding); on those hosts direct nvidia-smi over SSH is expected to " + "fail and srun is the authoritative GPU test" % overrides, ) ok = all(c["ok"] for c in checks)