diff --git a/roles/prometheus-slurm-exporter/defaults/main.yml b/roles/prometheus-slurm-exporter/defaults/main.yml index c737334ae..83f4841b8 100644 --- a/roles/prometheus-slurm-exporter/defaults/main.yml +++ b/roles/prometheus-slurm-exporter/defaults/main.yml @@ -1,4 +1,9 @@ -slurm_exporter_container: "deepops/prometheus-slurm-exporter:latest" +# Built locally from the bundled Dockerfile by default; set +# slurm_exporter_build_image: false and point slurm_exporter_container at a +# site image to use a registry instead. +slurm_exporter_container: "deepops/prometheus-slurm-exporter:0.20-1" +slurm_exporter_build_image: true +slurm_exporter_build_dir: /opt/deepops/build/slurm-exporter slurm_exporter_svc_name: "docker.slurm-exporter.service" slurm_exporter_state: started slurm_exporter_enabled: yes diff --git a/roles/prometheus-slurm-exporter/files/docker/Dockerfile b/roles/prometheus-slurm-exporter/files/docker/Dockerfile new file mode 100644 index 000000000..69caec384 --- /dev/null +++ b/roles/prometheus-slurm-exporter/files/docker/Dockerfile @@ -0,0 +1,34 @@ +# Prometheus Slurm exporter image, built locally by the +# prometheus-slurm-exporter role (no registry pull required). +# +# The container bind-mounts the host's Slurm client binaries and libraries, +# so the runtime stage provides a current glibc plus the system libraries +# those binaries link that are not part of the Slurm install prefix. +ARG SLURM_EXPORTER_BASE_IMAGE=ubuntu:24.04 + +FROM golang:1.24 AS build +ARG SLURM_EXPORTER_VERSION=0.20 +# The sed below fixes an upstream parsing bug present through vpenso master: +# the node collector calls sinfo -O without field widths, so node names of +# 20+ characters truncate into the next column and the exporter panics with +# "index out of range". The ": " width suffix disables truncation. +RUN git clone --depth 1 --branch ${SLURM_EXPORTER_VERSION} \ + https://github.com/vpenso/prometheus-slurm-exporter /src \ + && cd /src \ + && sed -i 's/NodeList,AllocMem,Memory,CPUsState,StateLong/NodeList: ,AllocMem: ,Memory: ,CPUsState: ,StateLong: /' node.go \ + && grep -q 'NodeList: ' node.go \ + && CGO_ENABLED=0 go build -o /prometheus-slurm-exporter + +FROM ${SLURM_EXPORTER_BASE_IMAGE} +# The slurm system user must resolve inside the container or the mounted +# Slurm client tools refuse to parse slurm.conf (SlurmUser lookup). +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libmunge2 \ + libjson-c5 \ + libyaml-0-2 \ + liblz4-1 \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --no-create-home --shell /usr/sbin/nologin slurm +COPY --from=build /prometheus-slurm-exporter /usr/local/bin/prometheus-slurm-exporter +ENTRYPOINT ["/usr/local/bin/prometheus-slurm-exporter"] diff --git a/roles/prometheus-slurm-exporter/tasks/main.yml b/roles/prometheus-slurm-exporter/tasks/main.yml index 397ea82e8..37b8e0dc3 100644 --- a/roles/prometheus-slurm-exporter/tasks/main.yml +++ b/roles/prometheus-slurm-exporter/tasks/main.yml @@ -43,6 +43,35 @@ - slurm-dashboard.json notify: restart grafana +- name: create image build dir + file: + path: "{{ slurm_exporter_build_dir }}" + state: directory + mode: "0755" + when: slurm_exporter_build_image + +- name: copy image build context + copy: + src: docker/Dockerfile + dest: "{{ slurm_exporter_build_dir }}/Dockerfile" + mode: "0644" + register: slurm_exporter_dockerfile + when: slurm_exporter_build_image + +- name: check for existing exporter image + command: docker image inspect {{ slurm_exporter_container }} + register: slurm_exporter_image_check + failed_when: false + changed_when: false + when: slurm_exporter_build_image + +- name: build exporter image + command: docker build -t {{ slurm_exporter_container }} {{ slurm_exporter_build_dir }} + when: > + slurm_exporter_build_image and + (slurm_exporter_image_check.rc != 0 or slurm_exporter_dockerfile.changed) + notify: restart slurm-exporter + - name: install systemd unit file template: src: templates/docker.slurm-exporter.service.j2 diff --git a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 index 31f6cf9fc..7c74e7906 100644 --- a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 +++ b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 @@ -8,8 +8,10 @@ TimeoutStartSec=0 Restart=always ExecStartPre=-/usr/bin/docker stop %n ExecStartPre=-/usr/bin/docker rm %n +{% if not slurm_exporter_build_image %} ExecStartPre=/usr/bin/docker pull {{ slurm_exporter_container }} -ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib/slurm:{{ slurm_install_prefix }}/lib/slurm:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} +{% endif %} +ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v {{ slurm_install_prefix }}/bin/sshare:{{ slurm_install_prefix }}/bin/sshare -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} [Install] WantedBy=multi-user.target