diff --git a/roles/prometheus-slurm-exporter/defaults/main.yml b/roles/prometheus-slurm-exporter/defaults/main.yml index 83f4841b8..8466dffbe 100644 --- a/roles/prometheus-slurm-exporter/defaults/main.yml +++ b/roles/prometheus-slurm-exporter/defaults/main.yml @@ -1,7 +1,7 @@ # Built locally from the bundled Dockerfile by default; set # slurm_exporter_build_image: false and point slurm_exporter_container at a # site image to use a registry instead. -slurm_exporter_container: "deepops/prometheus-slurm-exporter:0.20-1" +slurm_exporter_container: "deepops/prometheus-slurm-exporter:2.0.0" slurm_exporter_build_image: true slurm_exporter_build_dir: /opt/deepops/build/slurm-exporter slurm_exporter_svc_name: "docker.slurm-exporter.service" diff --git a/roles/prometheus-slurm-exporter/files/docker/Dockerfile b/roles/prometheus-slurm-exporter/files/docker/Dockerfile index 69caec384..f4dcdcc9f 100644 --- a/roles/prometheus-slurm-exporter/files/docker/Dockerfile +++ b/roles/prometheus-slurm-exporter/files/docker/Dockerfile @@ -1,23 +1,15 @@ -# Prometheus Slurm exporter image, built locally by the -# prometheus-slurm-exporter role (no registry pull required). +# DeepOps-owned Slurm exporter image, built locally by the +# prometheus-slurm-exporter role (no registry pull, no external source). # -# The container bind-mounts the host's Slurm client binaries and libraries, -# so the runtime stage provides a current glibc plus the system libraries -# those binaries link that are not part of the Slurm install prefix. +# The exporter source lives in this build context (see ../exporter). The +# container bind-mounts the host's Slurm client binaries and libraries, so +# the runtime stage provides a current glibc plus the system libraries those +# binaries link that are not part of the Slurm install prefix. ARG SLURM_EXPORTER_BASE_IMAGE=ubuntu:24.04 FROM golang:1.24 AS build -ARG SLURM_EXPORTER_VERSION=0.20 -# The sed below fixes an upstream parsing bug present through vpenso master: -# the node collector calls sinfo -O without field widths, so node names of -# 20+ characters truncate into the next column and the exporter panics with -# "index out of range". The ": " width suffix disables truncation. -RUN git clone --depth 1 --branch ${SLURM_EXPORTER_VERSION} \ - https://github.com/vpenso/prometheus-slurm-exporter /src \ - && cd /src \ - && sed -i 's/NodeList,AllocMem,Memory,CPUsState,StateLong/NodeList: ,AllocMem: ,Memory: ,CPUsState: ,StateLong: /' node.go \ - && grep -q 'NodeList: ' node.go \ - && CGO_ENABLED=0 go build -o /prometheus-slurm-exporter +COPY exporter/ /src/ +RUN cd /src && CGO_ENABLED=0 go build -buildvcs=false -o /deepops-slurm-exporter . FROM ${SLURM_EXPORTER_BASE_IMAGE} # The slurm system user must resolve inside the container or the mounted @@ -30,5 +22,5 @@ RUN apt-get update \ liblz4-1 \ && rm -rf /var/lib/apt/lists/* \ && useradd --system --no-create-home --shell /usr/sbin/nologin slurm -COPY --from=build /prometheus-slurm-exporter /usr/local/bin/prometheus-slurm-exporter -ENTRYPOINT ["/usr/local/bin/prometheus-slurm-exporter"] +COPY --from=build /deepops-slurm-exporter /usr/local/bin/deepops-slurm-exporter +ENTRYPOINT ["/usr/local/bin/deepops-slurm-exporter"] diff --git a/roles/prometheus-slurm-exporter/files/exporter/go.mod b/roles/prometheus-slurm-exporter/files/exporter/go.mod new file mode 100644 index 000000000..1eb8baa36 --- /dev/null +++ b/roles/prometheus-slurm-exporter/files/exporter/go.mod @@ -0,0 +1,3 @@ +module github.com/NVIDIA/deepops/roles/prometheus-slurm-exporter/files/exporter + +go 1.22 diff --git a/roles/prometheus-slurm-exporter/files/exporter/main.go b/roles/prometheus-slurm-exporter/files/exporter/main.go new file mode 100644 index 000000000..2029584ee --- /dev/null +++ b/roles/prometheus-slurm-exporter/files/exporter/main.go @@ -0,0 +1,258 @@ +// deepops-slurm-exporter exposes Slurm cluster state as Prometheus metrics. +// +// It is a dependency-free replacement for the previously used third-party +// exporter, emitting the metric names consumed by the DeepOps Grafana +// dashboard (slurm_nodes_*, slurm_queue_*, slurm_scheduler_*) plus the +// aggregate CPU gauges (slurm_cpus_*). Metrics are collected by executing +// the Slurm client tools on each scrape, with explicit no-truncation output +// formats so long node names and non-C locales cannot corrupt parsing. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "os/exec" + "sort" + "strconv" + "strings" + "time" +) + +var commandTimeout = 30 * time.Second + +// runCommand executes a Slurm client tool and returns its stdout. +func runCommand(name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).Output() + if err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return string(out), nil +} + +// ParseNodeStates aggregates `sinfo -h -o %T,%D` output into the dashboard's +// node state gauges. State names may carry suffix flags (*, ~, #, $, @, +) +// and long forms (allocated, drained); matching is by prefix category. +func ParseNodeStates(input string) map[string]float64 { + states := map[string]float64{ + "alloc": 0, "comp": 0, "down": 0, "drain": 0, "err": 0, + "fail": 0, "idle": 0, "maint": 0, "mix": 0, + } + for _, line := range strings.Split(input, "\n") { + parts := strings.Split(strings.TrimSpace(line), ",") + if len(parts) != 2 { + continue + } + state := strings.ToLower(strings.TrimRight(parts[0], "*~#$@+^-")) + count, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + continue + } + switch { + case strings.HasPrefix(state, "alloc"): + states["alloc"] += count + case strings.HasPrefix(state, "comp"): + states["comp"] += count + case strings.HasPrefix(state, "down"): + states["down"] += count + case strings.HasPrefix(state, "drain"): + states["drain"] += count + case strings.HasPrefix(state, "err"): + states["err"] += count + case strings.HasPrefix(state, "fail"): + states["fail"] += count + case strings.HasPrefix(state, "idle"): + states["idle"] += count + case strings.HasPrefix(state, "maint"): + states["maint"] += count + case strings.HasPrefix(state, "mix"): + states["mix"] += count + } + } + return states +} + +// ParseCPUs parses `sinfo -h -o %C` (allocated/idle/other/total). +func ParseCPUs(input string) (map[string]float64, error) { + fields := strings.Split(strings.TrimSpace(input), "/") + if len(fields) != 4 { + return nil, fmt.Errorf("unexpected %%C output: %q", strings.TrimSpace(input)) + } + keys := []string{"alloc", "idle", "other", "total"} + cpus := map[string]float64{} + for i, k := range keys { + v, err := strconv.ParseFloat(fields[i], 64) + if err != nil { + return nil, fmt.Errorf("unexpected %%C field %q: %w", fields[i], err) + } + cpus[k] = v + } + return cpus, nil +} + +// ParseQueueStates counts `squeue -a -r -h -o %T --states=all` job states +// into the dashboard's queue gauges. +func ParseQueueStates(input string) map[string]float64 { + queue := map[string]float64{ + "pending": 0, "running": 0, "suspended": 0, "cancelled": 0, + "completing": 0, "completed": 0, "failed": 0, "timeout": 0, + "node_fail": 0, "preempted": 0, + } + for _, line := range strings.Split(input, "\n") { + state := strings.ToLower(strings.TrimSpace(line)) + if state == "" { + continue + } + if _, ok := queue[state]; ok { + queue[state]++ + } + } + return queue +} + +// ParseScheduler extracts the dashboard's scheduler gauges from `sdiag` +// text output. Main and backfilling sections repeat the same field names, +// so section context is tracked explicitly. +func ParseScheduler(input string) map[string]float64 { + // Pre-initialize every gauge: on an idle cluster sdiag omits the mean + // and depth lines entirely (no cycles yet), and the dashboard expects + // the series to exist with value 0 rather than be absent. + sched := map[string]float64{ + "threads": 0, "queue_size": 0, "last_cycle": 0, "mean_cycle": 0, + "backfill_last_cycle": 0, "backfill_mean_cycle": 0, "backfill_depth_mean": 0, + } + backfill := false + numberAfterColon := func(line string) (float64, bool) { + idx := strings.LastIndex(line, ":") + if idx < 0 { + return 0, false + } + fields := strings.Fields(line[idx+1:]) + if len(fields) == 0 { + return 0, false + } + v, err := strconv.ParseFloat(fields[0], 64) + if err != nil { + return 0, false + } + return v, true + } + for _, line := range strings.Split(input, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "Backfilling stats") { + backfill = true + } + switch { + case strings.HasPrefix(trimmed, "Server thread count:"): + if v, ok := numberAfterColon(trimmed); ok { + sched["threads"] = v + } + case strings.HasPrefix(trimmed, "Agent queue size:"): + if v, ok := numberAfterColon(trimmed); ok { + sched["queue_size"] = v + } + case strings.HasPrefix(trimmed, "Last cycle:"): + if v, ok := numberAfterColon(trimmed); ok { + if backfill { + sched["backfill_last_cycle"] = v + } else { + sched["last_cycle"] = v + } + } + case strings.HasPrefix(trimmed, "Mean cycle:"): + if v, ok := numberAfterColon(trimmed); ok { + if backfill { + sched["backfill_mean_cycle"] = v + } else { + sched["mean_cycle"] = v + } + } + case strings.HasPrefix(trimmed, "Depth Mean:"): + if v, ok := numberAfterColon(trimmed); ok { + sched["backfill_depth_mean"] = v + } + } + } + return sched +} + +type metricsBuilder struct { + b strings.Builder +} + +func (m *metricsBuilder) gauge(name, help string, value float64) { + fmt.Fprintf(&m.b, "# HELP %s %s\n# TYPE %s gauge\n%s %g\n", name, help, name, name, value) +} + +func (m *metricsBuilder) gaugeSet(prefix, help string, values map[string]float64) { + keys := make([]string, 0, len(values)) + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + m.gauge(prefix+k, help+" ("+k+")", values[k]) + } +} + +func (m *metricsBuilder) String() string { return m.b.String() } + +// collect gathers all metrics; collector failures are reported through +// slurm_exporter_collector_errors rather than failing the whole scrape. +func collect() string { + m := &metricsBuilder{} + errors := 0 + + if out, err := runCommand("sinfo", "-h", "-o", "%T,%D"); err == nil { + m.gaugeSet("slurm_nodes_", "Count of nodes by state", ParseNodeStates(out)) + } else { + log.Printf("node collector: %v", err) + errors++ + } + + if out, err := runCommand("sinfo", "-h", "-o", "%C"); err == nil { + if cpus, perr := ParseCPUs(out); perr == nil { + m.gaugeSet("slurm_cpus_", "Aggregate CPUs by allocation state", cpus) + } else { + log.Printf("cpu collector: %v", perr) + errors++ + } + } else { + log.Printf("cpu collector: %v", err) + errors++ + } + + if out, err := runCommand("squeue", "-a", "-r", "-h", "-o", "%T", "--states=all"); err == nil { + m.gaugeSet("slurm_queue_", "Count of jobs by state", ParseQueueStates(out)) + } else { + log.Printf("queue collector: %v", err) + errors++ + } + + if out, err := runCommand("sdiag"); err == nil { + m.gaugeSet("slurm_scheduler_", "Slurm scheduler statistics", ParseScheduler(out)) + } else { + log.Printf("scheduler collector: %v", err) + errors++ + } + + m.gauge("slurm_exporter_collector_errors", + "Number of collectors that failed during this scrape", float64(errors)) + return m.String() +} + +func main() { + listen := flag.String("listen-address", ":8080", "address to serve /metrics on") + flag.Parse() + + http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + fmt.Fprint(w, collect()) + }) + log.Printf("deepops-slurm-exporter listening on %s", *listen) + log.Fatal(http.ListenAndServe(*listen, nil)) +} diff --git a/roles/prometheus-slurm-exporter/files/exporter/main_test.go b/roles/prometheus-slurm-exporter/files/exporter/main_test.go new file mode 100644 index 000000000..661653edf --- /dev/null +++ b/roles/prometheus-slurm-exporter/files/exporter/main_test.go @@ -0,0 +1,114 @@ +package main + +import "testing" + +func TestParseNodeStates(t *testing.T) { + input := "allocated,3\nidle,10\nmixed,2\ndrained*,1\ndown~,4\nmaint,1\n\nnot-a-state-line\n" + states := ParseNodeStates(input) + expect := map[string]float64{ + "alloc": 3, "idle": 10, "mix": 2, "drain": 1, "down": 4, + "maint": 1, "comp": 0, "err": 0, "fail": 0, + } + for k, v := range expect { + if states[k] != v { + t.Errorf("state %s: got %v want %v", k, states[k], v) + } + } +} + +func TestParseNodeStatesLongNamesIrrelevant(t *testing.T) { + // State aggregation uses %T,%D so node name length can never matter, + // but flag suffixes on states must still strip. + states := ParseNodeStates("idle*,5\nallocated#,2\n") + if states["idle"] != 5 || states["alloc"] != 2 { + t.Errorf("suffix stripping failed: %+v", states) + } +} + +func TestParseCPUs(t *testing.T) { + cpus, err := ParseCPUs("3/13/0/16\n") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cpus["alloc"] != 3 || cpus["idle"] != 13 || cpus["other"] != 0 || cpus["total"] != 16 { + t.Errorf("bad parse: %+v", cpus) + } + if _, err := ParseCPUs("garbage"); err == nil { + t.Error("expected error on malformed input") + } +} + +func TestParseQueueStates(t *testing.T) { + input := "PENDING\nPENDING\nRUNNING\nCOMPLETED\nFAILED\nNODE_FAIL\nUNKNOWN_STATE\n\n" + q := ParseQueueStates(input) + if q["pending"] != 2 || q["running"] != 1 || q["completed"] != 1 || + q["failed"] != 1 || q["node_fail"] != 1 || q["timeout"] != 0 { + t.Errorf("bad parse: %+v", q) + } +} + +func TestParseScheduler(t *testing.T) { + // Shape observed from sdiag on Slurm 26.05. + input := `******************************************************* +sdiag output at Wed Jul 15 17:48:55 2026 (1784137735) +Data since Wed Jul 15 17:28:49 2026 (1784136529) +******************************************************* +Server thread count: 3 +Agent queue size: 0 + +Main schedule statistics (microseconds): + Last cycle: 289 + Max cycle: 1691 + Total cycles: 25 + Mean cycle: 513 + +Backfilling stats + Total backfilled jobs (since last slurm start): 0 + Last cycle when: Wed Jul 15 17:47:21 2026 (1784137641) + Last cycle: 45 + Mean cycle: 60 + Last depth cycle: 0 + Depth Mean: 4 +` + s := ParseScheduler(input) + expect := map[string]float64{ + "threads": 3, "queue_size": 0, "last_cycle": 289, "mean_cycle": 513, + "backfill_last_cycle": 45, "backfill_mean_cycle": 60, "backfill_depth_mean": 4, + } + for k, v := range expect { + if s[k] != v { + t.Errorf("scheduler %s: got %v want %v", k, s[k], v) + } + } +} + +func TestParseSchedulerIdleClusterEmitsFullSet(t *testing.T) { + // Zero-cycle clusters omit Mean cycle / Depth Mean lines entirely; + // every scheduler gauge must still exist with value 0. + s := ParseScheduler("Server thread count: 3\nBackfilling stats\n\tTotal cycles: 0\n\tLast cycle: 0\n") + for _, k := range []string{"threads", "queue_size", "last_cycle", "mean_cycle", + "backfill_last_cycle", "backfill_mean_cycle", "backfill_depth_mean"} { + if _, ok := s[k]; !ok { + t.Errorf("missing scheduler gauge %s on idle cluster", k) + } + } +} + +func TestParseSchedulerLastCycleWhenNotConfused(t *testing.T) { + // "Last cycle when:" carries a timestamp and must not overwrite + // "Last cycle:". + s := ParseScheduler("Backfilling stats\n\tLast cycle when: Wed Jul 15 17:47:21 2026 (1784137641)\n\tLast cycle: 45\n") + if s["backfill_last_cycle"] != 45 { + t.Errorf("Last cycle when: leaked into backfill_last_cycle: %+v", s) + } +} + +func TestMetricsOutputFormat(t *testing.T) { + m := &metricsBuilder{} + m.gauge("slurm_nodes_idle", "Count of nodes by state (idle)", 5) + out := m.String() + want := "# HELP slurm_nodes_idle Count of nodes by state (idle)\n# TYPE slurm_nodes_idle gauge\nslurm_nodes_idle 5\n" + if out != want { + t.Errorf("exposition format mismatch:\ngot: %q\nwant: %q", out, want) + } +} diff --git a/roles/prometheus-slurm-exporter/tasks/main.yml b/roles/prometheus-slurm-exporter/tasks/main.yml index 37b8e0dc3..a8ef4e0ce 100644 --- a/roles/prometheus-slurm-exporter/tasks/main.yml +++ b/roles/prometheus-slurm-exporter/tasks/main.yml @@ -58,6 +58,14 @@ register: slurm_exporter_dockerfile when: slurm_exporter_build_image +- name: copy exporter source + copy: + src: exporter/ + dest: "{{ slurm_exporter_build_dir }}/exporter/" + mode: "0644" + register: slurm_exporter_source + when: slurm_exporter_build_image + - name: check for existing exporter image command: docker image inspect {{ slurm_exporter_container }} register: slurm_exporter_image_check @@ -69,7 +77,7 @@ command: docker build -t {{ slurm_exporter_container }} {{ slurm_exporter_build_dir }} when: > slurm_exporter_build_image and - (slurm_exporter_image_check.rc != 0 or slurm_exporter_dockerfile.changed) + (slurm_exporter_image_check.rc != 0 or slurm_exporter_dockerfile.changed or slurm_exporter_source.changed) notify: restart slurm-exporter - name: install systemd unit file diff --git a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 index 0b4d5fb28..502c7bf3c 100644 --- a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 +++ b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 @@ -12,7 +12,7 @@ ExecStartPre=-/usr/bin/docker rm %n {% if not slurm_exporter_build_image %} ExecStartPre=/usr/bin/docker pull {{ slurm_exporter_container }} {% endif %} -ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v {{ slurm_install_prefix }}/bin/sshare:{{ slurm_install_prefix }}/bin/sshare -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} +ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} [Install] WantedBy=multi-user.target