From 59fa7a007e4446b890e046ffbf6de5e3607ed1ae Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 15 Jul 2026 10:04:45 -0600 Subject: [PATCH] fix: build the Slurm exporter image locally with a current base The role pulled deepops/prometheus-slurm-exporter:latest, a 2020-era image whose glibc is older than the Slurm client binaries bind-mounted from modern hosts, so sinfo/squeue/sdiag failed inside the container and the service crash-looped. The tag was also unpinned and the image had no in-repo build path. Build the image locally from a bundled Dockerfile instead: the vpenso prometheus-slurm-exporter 0.20 binary (matching the metric names used by the shipped Grafana dashboard) on an Ubuntu 24.04 base with the system libraries the mounted Slurm binaries link. The image tag is pinned, no registry pull is required, and sites can still supply their own image with slurm_exporter_build_image: false. Mount the full Slurm install lib directory so libslurm resolves alongside the plugin dir. Fixes #1328 --- .../defaults/main.yml | 7 +++- .../files/docker/Dockerfile | 34 +++++++++++++++++++ .../prometheus-slurm-exporter/tasks/main.yml | 29 ++++++++++++++++ .../docker.slurm-exporter.service.j2 | 4 ++- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 roles/prometheus-slurm-exporter/files/docker/Dockerfile diff --git a/roles/prometheus-slurm-exporter/defaults/main.yml b/roles/prometheus-slurm-exporter/defaults/main.yml index c737334ae..83f4841b8 100644 --- a/roles/prometheus-slurm-exporter/defaults/main.yml +++ b/roles/prometheus-slurm-exporter/defaults/main.yml @@ -1,4 +1,9 @@ -slurm_exporter_container: "deepops/prometheus-slurm-exporter:latest" +# Built locally from the bundled Dockerfile by default; set +# slurm_exporter_build_image: false and point slurm_exporter_container at a +# site image to use a registry instead. +slurm_exporter_container: "deepops/prometheus-slurm-exporter:0.20-1" +slurm_exporter_build_image: true +slurm_exporter_build_dir: /opt/deepops/build/slurm-exporter slurm_exporter_svc_name: "docker.slurm-exporter.service" slurm_exporter_state: started slurm_exporter_enabled: yes diff --git a/roles/prometheus-slurm-exporter/files/docker/Dockerfile b/roles/prometheus-slurm-exporter/files/docker/Dockerfile new file mode 100644 index 000000000..69caec384 --- /dev/null +++ b/roles/prometheus-slurm-exporter/files/docker/Dockerfile @@ -0,0 +1,34 @@ +# Prometheus Slurm exporter image, built locally by the +# prometheus-slurm-exporter role (no registry pull required). +# +# The container bind-mounts the host's Slurm client binaries and libraries, +# so the runtime stage provides a current glibc plus the system libraries +# those binaries link that are not part of the Slurm install prefix. +ARG SLURM_EXPORTER_BASE_IMAGE=ubuntu:24.04 + +FROM golang:1.24 AS build +ARG SLURM_EXPORTER_VERSION=0.20 +# The sed below fixes an upstream parsing bug present through vpenso master: +# the node collector calls sinfo -O without field widths, so node names of +# 20+ characters truncate into the next column and the exporter panics with +# "index out of range". The ": " width suffix disables truncation. +RUN git clone --depth 1 --branch ${SLURM_EXPORTER_VERSION} \ + https://github.com/vpenso/prometheus-slurm-exporter /src \ + && cd /src \ + && sed -i 's/NodeList,AllocMem,Memory,CPUsState,StateLong/NodeList: ,AllocMem: ,Memory: ,CPUsState: ,StateLong: /' node.go \ + && grep -q 'NodeList: ' node.go \ + && CGO_ENABLED=0 go build -o /prometheus-slurm-exporter + +FROM ${SLURM_EXPORTER_BASE_IMAGE} +# The slurm system user must resolve inside the container or the mounted +# Slurm client tools refuse to parse slurm.conf (SlurmUser lookup). +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libmunge2 \ + libjson-c5 \ + libyaml-0-2 \ + liblz4-1 \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --no-create-home --shell /usr/sbin/nologin slurm +COPY --from=build /prometheus-slurm-exporter /usr/local/bin/prometheus-slurm-exporter +ENTRYPOINT ["/usr/local/bin/prometheus-slurm-exporter"] diff --git a/roles/prometheus-slurm-exporter/tasks/main.yml b/roles/prometheus-slurm-exporter/tasks/main.yml index 397ea82e8..37b8e0dc3 100644 --- a/roles/prometheus-slurm-exporter/tasks/main.yml +++ b/roles/prometheus-slurm-exporter/tasks/main.yml @@ -43,6 +43,35 @@ - slurm-dashboard.json notify: restart grafana +- name: create image build dir + file: + path: "{{ slurm_exporter_build_dir }}" + state: directory + mode: "0755" + when: slurm_exporter_build_image + +- name: copy image build context + copy: + src: docker/Dockerfile + dest: "{{ slurm_exporter_build_dir }}/Dockerfile" + mode: "0644" + register: slurm_exporter_dockerfile + when: slurm_exporter_build_image + +- name: check for existing exporter image + command: docker image inspect {{ slurm_exporter_container }} + register: slurm_exporter_image_check + failed_when: false + changed_when: false + when: slurm_exporter_build_image + +- name: build exporter image + command: docker build -t {{ slurm_exporter_container }} {{ slurm_exporter_build_dir }} + when: > + slurm_exporter_build_image and + (slurm_exporter_image_check.rc != 0 or slurm_exporter_dockerfile.changed) + notify: restart slurm-exporter + - name: install systemd unit file template: src: templates/docker.slurm-exporter.service.j2 diff --git a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 index 31f6cf9fc..7c74e7906 100644 --- a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 +++ b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 @@ -8,8 +8,10 @@ TimeoutStartSec=0 Restart=always ExecStartPre=-/usr/bin/docker stop %n ExecStartPre=-/usr/bin/docker rm %n +{% if not slurm_exporter_build_image %} ExecStartPre=/usr/bin/docker pull {{ slurm_exporter_container }} -ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib/slurm:{{ slurm_install_prefix }}/lib/slurm:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} +{% endif %} +ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v {{ slurm_install_prefix }}/bin/sshare:{{ slurm_install_prefix }}/bin/sshare -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} [Install] WantedBy=multi-user.target