diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 782a41dfd..5dc9501e3 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -29,12 +29,13 @@ - [Copying Files](framework/copying_files.md) - [Running in Kubernetes](./framework/kubernetes.md) - [Observability Stack](framework/observability/observability_stack.md) - - [Overview](framework/observability/observability_stack.md) - [Metrics](framework/observability/metrics.md) - [Logs](framework/observability/logs.md) - [Profiling](framework/observability/profiling.md) - [PostgreSQL](framework/observability/postgresql.md) - [BlockScout](framework/observability/blockscout.md) +- [Observability Stack (VictoriaMetrics)](framework/observability-victoria/observability_stack_victoria.md) + - [Logs](framework/observability-victoria/logs.md) - [Components](framework/components/overview.md) - [Overview](framework/components/overview.md) - [Blockchains](framework/components/blockchains/overview.md) diff --git a/book/src/framework/observability-victoria/logs.md b/book/src/framework/observability-victoria/logs.md new file mode 100644 index 000000000..b668680bf --- /dev/null +++ b/book/src/framework/observability-victoria/logs.md @@ -0,0 +1,43 @@ +# Logs + +WASP load data is stored in `VictoriaLogs` and queried with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). The bundled **WASP (VictoriaLogs)** dashboard is loaded automatically, but you can also explore the raw data in [localhost:3000](http://localhost:3000/explore) (`VictoriaLogs` datasource) or directly via the VictoriaLogs UI at [localhost:9428](http://localhost:9428/select/vmui). + +Every WASP record carries these fields: `go_test_name`, `gen_name`, `call_group`, `branch`, `commit` and `test_data_type` (`stats` or `responses`). Numeric values (`current_rps`, `current_instances`, `duration`, ...) live in the JSON `_msg`, so add `| unpack_json` before aggregating them. + +[Explore](http://localhost:3000/explore?schemaVersion=1&panes=%7B%22il3%22:%7B%22datasource%22:%22victorialogs%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22victoriametrics-logs-datasource%22,%22uid%22:%22victorialogs%22%7D,%22editorMode%22:%22code%22,%22expr%22:%22%2A%22,%22queryType%22:%22instant%22%7D%5D,%22range%22:%7B%22from%22:%22now-5m%22,%22to%22:%22now%22%7D%7D%7D&orgId=1) + +## Example WASP Log Queries + +Queries: +- All data for a single test +```sql +go_test_name:="TestMyLoad" +``` +- Periodic generator stats (RPS, VUs, sampling) +```sql +go_test_name:="TestMyLoad" AND test_data_type:stats +``` +- Individual responses (one log line per call) +```sql +go_test_name:="TestMyLoad" AND test_data_type:responses +``` +- Current RPS per generator +```sql +test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_rps) value +``` +- Current VUs (virtual users) per generator +```sql +test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_instances) value +``` +- Failed responses +```sql +test_data_type:responses AND _msg:"\"failed\":true" +``` +- Timed out responses +```sql +test_data_type:responses AND _msg:"\"timeout\":true" +``` +- Latency quantiles (p99/p95/p50, in nanoseconds) per call group +```sql +test_data_type:responses | unpack_json | stats by (gen_name, call_group) quantile(0.99, duration) value +``` diff --git a/book/src/framework/observability-victoria/metrics.md b/book/src/framework/observability-victoria/metrics.md new file mode 100644 index 000000000..b05321b8f --- /dev/null +++ b/book/src/framework/observability-victoria/metrics.md @@ -0,0 +1 @@ +# Metrics diff --git a/book/src/framework/observability-victoria/observability_stack_victoria.md b/book/src/framework/observability-victoria/observability_stack_victoria.md new file mode 100644 index 000000000..3ddd0c988 --- /dev/null +++ b/book/src/framework/observability-victoria/observability_stack_victoria.md @@ -0,0 +1,36 @@ +# Local Observability Stack (VictoriaMetrics) + +Minimal setup for experimenting with OTEL metrics & logs in Grafana. + +## Components + +| Service | Port | Purpose | +|---|---|---| +| Grafana | 3000 | UI, anonymous admin enabled | +| OTEL Collector | 4317 (gRPC), 4318 (HTTP) | Receives OTLP from your app | +| VictoriaMetrics | 8428 | Metrics TSDB (Prom remote_write in, MetricsQL out) | +| VictoriaLogs | 9428 | Logs DB (OTLP in, LogsQL out) | + +## Run + +```bash +# start the observability stack +ctf obs up -vm +# remove the stack with all the data (volumes) +ctf obs d -vm +# restart the stack removing all the data (volumes) +ctf obs r -vm +``` + +## Developing + +Change compose files under `framework/cmd/observability` and restart the stack (removing volumes too) +``` +just reload-cli && ctf obs r +``` + +## Local Dashboards (Docker) + +You can create a dashboard using [UI](http://localhost:3000) and put them under `$pwd/dashboards` folder then commit, they'll be loaded automatically on start and you can find them [here](http://localhost:3000/dashboards) under `local` directory. + +`$pwd` is you current working directory from which you call `ctf obs u` diff --git a/framework/.changeset/v0.16.4.md b/framework/.changeset/v0.16.4.md new file mode 100644 index 000000000..4af4aa0d4 --- /dev/null +++ b/framework/.changeset/v0.16.4.md @@ -0,0 +1 @@ +- VictoriaMetrics stack in CTF, docs \ No newline at end of file diff --git a/framework/cmd/main.go b/framework/cmd/main.go index e4f7aa037..f4493bc48 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -19,6 +19,7 @@ import ( func main() { app := &cli.App{ + Version: "v0.16.4", Name: "ctf", Usage: "Chainlink Testing Framework CLI", UsageText: "'ctf' is a useful utility that can:\n- clean up test docker containers\n- modify test files\n- create a local observability stack with Grafana/Loki/Pyroscope", @@ -535,12 +536,21 @@ Be aware that any TODO requires your attention before your run the final test! Usage: "Spin up all the observability services", Value: false, }, + &cli.BoolFlag{ + Name: "victoria", + Aliases: []string{"vm"}, + Usage: "Spin up all the observability services (VictoriaMetrics)", + Value: false, + }, }, Description: "Spins up a local observability stack. Has two modes, standard (Loki, Prometheus, Grafana and OTEL) and full including also Tempo, Cadvisor and PostgreSQL metrics", Action: func(c *cli.Context) error { if c.Bool("full") { return framework.ObservabilityUpFull() } + if c.Bool("victoria") { + return framework.ObservabilityVictoriaMetricsUp() + } return framework.ObservabilityUp() }, }, @@ -555,9 +565,20 @@ Be aware that any TODO requires your attention before your run the final test! Usage: "Removes all the observability services (this flag exists for compatibility, all the services are always removed with 'down')", Value: false, }, + &cli.BoolFlag{ + Name: "victoria", + Aliases: []string{"vm"}, + Usage: "Spin up all the observability services (VictoriaMetrics)", + Value: false, + }, }, Description: "Removes local observability stack", - Action: func(c *cli.Context) error { return framework.ObservabilityDown() }, + Action: func(c *cli.Context) error { + if c.Bool("victoria") { + return framework.ObservabilityVictoriaDown() + } + return framework.ObservabilityDown() + }, }, { Name: "restart", @@ -570,16 +591,30 @@ Be aware that any TODO requires your attention before your run the final test! Usage: "Restart all observability services (this flag exists for compatibility, all the services are always removed with 'down')", Value: false, }, + &cli.BoolFlag{ + Name: "victoria", + Aliases: []string{"vm"}, + Usage: "Spin up all the observability services (VictoriaMetrics)", + Value: false, + }, }, Description: "Restart a local observability stack", Action: func(c *cli.Context) error { - // always remove all the containers and volumes to clean up the data - if err := framework.ObservabilityDown(); err != nil { - return err + if c.Bool("victoria") { + if err := framework.ObservabilityVictoriaDown(); err != nil { + return err + } + } else { + if err := framework.ObservabilityDown(); err != nil { + return err + } } if c.Bool("full") { return framework.ObservabilityUpFull() } + if c.Bool("victoria") { + return framework.ObservabilityVictoriaMetricsUp() + } return framework.ObservabilityUp() }, }, diff --git a/framework/observability.go b/framework/observability.go index 55003116c..faa8fc326 100644 --- a/framework/observability.go +++ b/framework/observability.go @@ -14,6 +14,13 @@ import ( var EmbeddedObservabilityFiles embed.FS const ( + LGTMDockerComposePath = "compose" + VictoriaMetricsDockerComposePath = "compose-victoria-metrics" + + LocalVictoriaMetrics = "http://localhost:8428" + LocalVictoriaLogs = "http://localhost:9428" + LocalOTELCollectorHTTP = "http://localhost:4318" + LocalOTELCollectorgRPC = "http://localhost:4317" LocalGrafanaBaseURL = "http://localhost:3000" LocalLokiBaseURL = "http://localhost:3030" LocalPrometheusBaseURL = "http://localhost:9099" @@ -181,7 +188,7 @@ func ObservabilityUpOnlyLoki() error { if err != nil { return err } - composeDir := filepath.Join(obsDir, "compose") + composeDir := filepath.Join(obsDir, LGTMDockerComposePath) _ = DefaultNetwork(nil) if err := NewPromtail(); err != nil { return err @@ -208,7 +215,7 @@ func ObservabilityUp() error { if err != nil { return err } - composeDir := filepath.Join(obsDir, "compose") + composeDir := filepath.Join(obsDir, LGTMDockerComposePath) _ = DefaultNetwork(nil) if err := NewPromtail(); err != nil { return err @@ -228,6 +235,34 @@ func ObservabilityUp() error { return nil } +// ObservabilityVictoriaMetricsUp VictoriaMetrics stack for load testing and performance investigations +func ObservabilityVictoriaMetricsUp() error { + L.Info().Msg("Creating local observability stack (VictoriaMetrics)") + if err := extractAllFiles("observability"); err != nil { + return err + } + obsDir, err := getObservabilityDir() + if err != nil { + return err + } + composeDir := filepath.Join(obsDir, VictoriaMetricsDockerComposePath) + _ = DefaultNetwork(nil) + err = RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose up -d + `, composeDir)) + if err != nil { + return err + } + fmt.Println() + L.Info().Msgf("Grafana: %s", LocalGrafanaBaseURL) + L.Info().Msgf("OTEL Collector HTTP: %s", LocalOTELCollectorHTTP) + L.Info().Msgf("OTEL Collector gRPC: %s", LocalOTELCollectorgRPC) + L.Info().Msgf("VictoriaMetrics: %s", LocalVictoriaMetrics) + L.Info().Msgf("VictoriaLogs: %s", LocalVictoriaLogs) + return nil +} + // ObservabilityUpFull full stack for load testing and performance investigations func ObservabilityUpFull() error { L.Info().Msg("Creating full local observability stack") @@ -238,7 +273,7 @@ func ObservabilityUpFull() error { if err != nil { return err } - composeDir := filepath.Join(obsDir, "compose") + composeDir := filepath.Join(obsDir, LGTMDockerComposePath) _ = DefaultNetwork(nil) if err := NewPromtail(); err != nil { return err @@ -260,13 +295,27 @@ func ObservabilityUpFull() error { return nil } +func ObservabilityVictoriaDown() error { + L.Info().Msg("Removing local observability stack (Victoria Metrics)") + obsDir, err := getObservabilityDir() + if err != nil { + return err + } + composeDir := filepath.Join(obsDir, VictoriaMetricsDockerComposePath) + _ = RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose down -v + `, composeDir)) + return nil +} + func ObservabilityDown() error { L.Info().Msg("Removing local observability stack") obsDir, err := getObservabilityDir() if err != nil { return err } - composeDir := filepath.Join(obsDir, "compose") + composeDir := filepath.Join(obsDir, LGTMDockerComposePath) _ = RunCommand("bash", "-c", fmt.Sprintf(` cd %s && \ docker compose down -v && docker rm -f promtail diff --git a/framework/observability/compose-victoria-metrics/README.md b/framework/observability/compose-victoria-metrics/README.md new file mode 100644 index 000000000..7ce3665ea --- /dev/null +++ b/framework/observability/compose-victoria-metrics/README.md @@ -0,0 +1,43 @@ +# Local VictoriaMetrics + VictoriaLogs + OTEL stack + +Minimal setup for experimenting with OTEL metrics & logs in Grafana. + +## Components + +| Service | Port | Purpose | +|---|---|---| +| Grafana | 3000 | UI, anonymous admin enabled | +| OTEL Collector | 4317 (gRPC), 4318 (HTTP) | Receives OTLP from your app | +| VictoriaMetrics | 8428 | Metrics TSDB (Prom remote_write in, MetricsQL out) | +| VictoriaLogs | 9428 | Logs DB (OTLP in, LogsQL out) | + +## Run + +```bash +docker compose up -d +``` + +Grafana: (no login required, anonymous admin). + +Both datasources (`VictoriaMetrics`, `VictoriaLogs`) are auto-provisioned. + +## Point your Go app at it + +In the OTEL exporter, use endpoint `localhost:4317` (gRPC) or `localhost:4318` (HTTP). Both metrics and logs go to the same collector — it fans them out to VM/VL. + +## Tear down + +```bash +docker compose down # keep data +docker compose down -v # wipe volumes +``` + +## Quick sanity checks + +```bash +# Metrics ingest +curl http://localhost:8428/api/v1/query?query=up + +# Logs ingest +curl 'http://localhost:9428/select/logsql/query?query=*&limit=10' +``` diff --git a/framework/observability/compose-victoria-metrics/docker-compose.yml b/framework/observability/compose-victoria-metrics/docker-compose.yml new file mode 100644 index 000000000..7c14c499b --- /dev/null +++ b/framework/observability/compose-victoria-metrics/docker-compose.yml @@ -0,0 +1,69 @@ +services: + victoriametrics: + image: victoriametrics/victoria-metrics:v1.106.1 + container_name: victoriametrics + ports: + - "8428:8428" + command: + - "--storageDataPath=/storage" + - "--retentionPeriod=7d" + - "--httpListenAddr=:8428" + volumes: + - vm-data:/storage + restart: unless-stopped + + victorialogs: + image: victoriametrics/victoria-logs:v0.40.0-victorialogs + container_name: victorialogs + ports: + - "9428:9428" + command: + - "--storageDataPath=/vlogs" + - "--httpListenAddr=:9428" + - "--retentionPeriod=7d" + volumes: + - vl-data:/vlogs + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1 + container_name: otel-collector + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + depends_on: + - victoriametrics + - victorialogs + restart: unless-stopped + + grafana: + image: grafana/grafana:11.3.1 + container_name: grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_INSTALL_PLUGINS=victoriametrics-logs-datasource + - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-logs-datasource + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + # Embedded internal dashboards (e.g. WASP for VictoriaLogs) + - ./grafana/dashboards:/etc/dashboards/internal:ro + # User dashboards from $pwd/dashboards (synced by copyLocalDashboards) + - ../dashboards:/etc/dashboards/local:ro + - grafana-data:/var/lib/grafana + depends_on: + - victoriametrics + - victorialogs + restart: unless-stopped + +volumes: + vm-data: + vl-data: + grafana-data: diff --git a/framework/observability/compose-victoria-metrics/grafana/dashboards/wasp.json b/framework/observability/compose-victoria-metrics/grafana/dashboards/wasp.json new file mode 100644 index 000000000..fca4daff1 --- /dev/null +++ b/framework/observability/compose-victoria-metrics/grafana/dashboards/wasp.json @@ -0,0 +1,418 @@ +{ + "annotations": {"list": []}, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 100, + "title": "WASP Load Stats", + "type": "row", + "panels": [] + }, + { + "id": 1, + "type": "stat", + "title": "RPS (Now)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name", + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_rps) value", + "legendFormat": "{{go_test_name}} {{gen_name}} RPS" + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "VUs (Now)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name" + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_instances) value", + "legendFormat": "{{go_test_name}} {{gen_name}} VUs" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Responses/sec (Now)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name" + }, + "targets": [ + { + "refId": "A", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:responses | stats by (go_test_name, gen_name) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} Responses/sec" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Successful requests (Total)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name" + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses AND NOT _msg:\"\\\"failed\\\":true\" | stats by (go_test_name, gen_name) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} Successful requests" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Errored requests (Total)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name", + "colorMode": "value" + }, + "fieldConfig": { + "defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}} + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses AND _msg:\"\\\"failed\\\":true\" | stats by (go_test_name, gen_name) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} Errored requests" + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Timed out requests (Total)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 1}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "horizontal", + "textMode": "value_and_name", + "colorMode": "value" + }, + "fieldConfig": { + "defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}} + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses AND _msg:\"\\\"timeout\\\":true\" | stats by (go_test_name, gen_name) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} Timed out requests" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "RPS/VUs per schedule segments", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 5}, + "fieldConfig": {"defaults": {"custom": {"lineInterpolation": "stepBefore", "fillOpacity": 10}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [ + { + "refId": "RPS", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_rps) value", + "legendFormat": "{{go_test_name}} {{gen_name}} RPS" + }, + { + "refId": "VUS", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(current_instances) value", + "legendFormat": "{{go_test_name}} {{gen_name}} VUs" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Responses/sec (Generator, CallGroup)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 5}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [ + { + "refId": "A", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses | stats by (go_test_name, gen_name, call_group) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} responses/sec" + }, + { + "refId": "B", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:responses | stats by (go_test_name, gen_name) count() value", + "legendFormat": "{{go_test_name}} {{gen_name}} Total responses/sec" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Latency quantiles over groups (99, 95, 50)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 13}, + "fieldConfig": {"defaults": {"unit": "ns"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [ + { + "refId": "Q99", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses | unpack_json | stats by (go_test_name, gen_name, call_group) quantile(0.99, duration) value", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} Q 99" + }, + { + "refId": "Q95", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses | unpack_json | stats by (go_test_name, gen_name, call_group) quantile(0.95, duration) value", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} Q 95" + }, + { + "refId": "Q50", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses | unpack_json | stats by (go_test_name, gen_name, call_group) quantile(0.5, duration) value", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} Q 50" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Responses latencies by types over time (Generator, CallGroup)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 13}, + "fieldConfig": {"defaults": {"unit": "ns"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [ + { + "refId": "A", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND call_group:~\"${call_group:pipe}\" AND test_data_type:responses | unpack_json | stats by (go_test_name, gen_name, call_group) max(duration) value", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}}" + }, + { + "refId": "B", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:responses | unpack_json | stats by (go_test_name, gen_name) max(duration) value", + "legendFormat": "{{go_test_name}} {{gen_name}} all groups" + } + ] + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 21}, + "id": 200, + "title": "WASP Debug", + "type": "row", + "panels": [] + }, + { + "id": 20, + "type": "stat", + "title": "Latest segment stats", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 6, "w": 24, "x": 0, "y": 22}, + "fieldConfig": {"defaults": {"unit": "decbytes"}}, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "graphMode": "area", + "textMode": "value_and_name", + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "queryType": "stats", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" | stats sum_len(_msg) value", + "legendFormat": "Overall logs size" + }, + { + "refId": "B", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" | stats sum_len(_msg) value", + "legendFormat": "Logs size per second" + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "CallResult sampling (successful results)", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "targets": [ + { + "refId": "REC", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(samples_recorded) value", + "legendFormat": "{{go_test_name}} {{gen_name}} recorded" + }, + { + "refId": "SKIP", + "queryType": "statsRange", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats | unpack_json | stats by (go_test_name, gen_name) max(samples_skipped) value", + "legendFormat": "{{go_test_name}} {{gen_name}} skipped" + } + ] + }, + { + "id": 30, + "type": "logs", + "title": "Stats logs", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 36}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, + "targets": [ + { + "refId": "A", + "queryType": "logs", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:stats" + } + ] + }, + { + "id": 31, + "type": "logs", + "title": "Failed responses", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, + "targets": [ + { + "refId": "A", + "queryType": "logs", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:responses AND _msg:\"\\\"failed\\\":true\"" + } + ] + }, + { + "id": 32, + "type": "logs", + "title": "Timed out responses", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, + "targets": [ + { + "refId": "A", + "queryType": "logs", + "expr": "go_test_name:~\"${go_test_name:pipe}\" AND gen_name:~\"${gen_name:pipe}\" AND test_data_type:responses AND _msg:\"\\\"timeout\\\":true\"" + } + ] + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["generated", "load-test", "wasp", "victorialogs"], + "templating": { + "list": [ + { + "name": "go_test_name", + "label": "Go test name", + "type": "query", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "query": {"editorMode": "builder", "type": "fieldValue", "field": "go_test_name"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 1 + }, + { + "name": "gen_name", + "label": "Generator name", + "type": "query", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "query": {"editorMode": "builder", "type": "fieldValue", "field": "gen_name"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 1 + }, + { + "name": "call_group", + "label": "Call group", + "type": "query", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "query": {"editorMode": "builder", "type": "fieldValue", "field": "call_group"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 1 + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "query": {"editorMode": "builder", "type": "fieldValue", "field": "branch"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 1 + }, + { + "name": "commit", + "label": "Commit", + "type": "query", + "datasource": {"type": "victoriametrics-logs-datasource", "uid": "victorialogs"}, + "query": {"editorMode": "builder", "type": "fieldValue", "field": "commit"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 1 + } + ] + }, + "time": {"from": "now-30m", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "WASP (VictoriaLogs)", + "uid": "wasp-victorialogs", + "version": 1, + "weekStart": "" +} diff --git a/framework/observability/compose-victoria-metrics/grafana/provisioning/dashboards/dashboards.yaml b/framework/observability/compose-victoria-metrics/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 000000000..25d6f19b6 --- /dev/null +++ b/framework/observability/compose-victoria-metrics/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: default + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/dashboards + foldersFromFilesStructure: true diff --git a/framework/observability/compose-victoria-metrics/grafana/provisioning/datasources/datasources.yaml b/framework/observability/compose-victoria-metrics/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 000000000..2368c2417 --- /dev/null +++ b/framework/observability/compose-victoria-metrics/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: VictoriaMetrics + uid: victoriametrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 + isDefault: true + editable: true + jsonData: + httpMethod: POST + timeInterval: "15s" + + - name: VictoriaLogs + uid: victorialogs + type: victoriametrics-logs-datasource + access: proxy + url: http://victorialogs:9428 + editable: true diff --git a/framework/observability/compose-victoria-metrics/otel/otel-collector-config.yaml b/framework/observability/compose-victoria-metrics/otel/otel-collector-config.yaml new file mode 100644 index 000000000..369654249 --- /dev/null +++ b/framework/observability/compose-victoria-metrics/otel/otel-collector-config.yaml @@ -0,0 +1,40 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 1024 + +exporters: + # VictoriaMetrics accepts Prometheus remote_write + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + + # VictoriaLogs accepts OTLP logs natively + otlphttp/logs: + endpoint: http://victorialogs:9428/insert/opentelemetry + tls: + insecure: true + + debug: + verbosity: basic + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheusremotewrite, debug] + + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/logs, debug] diff --git a/framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json b/framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json index 1b7dd0523..913904c1f 100644 --- a/framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json +++ b/framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json @@ -1,1432 +1,428 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": false, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, + "annotations": {"list": []}, "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 17447, + "graphTooltip": 0, + "id": null, "links": [], "liveNow": false, "panels": [ { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "P5DCFC7561CCDE821" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 16, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P5DCFC7561CCDE821" - }, - "refId": "A" - } - ], + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 100, "title": "WASP Load Stats", - "type": "row" + "type": "row", + "panels": [] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 1 - }, "id": 1, + "type": "stat", + "title": "RPS (Now)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, "textMode": "value_and_name", - "wideLayout": true + "colorMode": "value" }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_rps [1s]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} RPS", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_rps [10s]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)", + "legendFormat": "{{go_test_name}} {{gen_name}} RPS" } - ], - "title": "RPS (Now)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 1 - }, "id": 2, + "type": "stat", + "title": "VUs (Now)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, - "textMode": "value_and_name", - "wideLayout": true + "textMode": "value_and_name" }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "sum(max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_instances [$__range]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} VUs", - "refId": "A" + "refId": "A", + "queryType": "instant", + "expr": "max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_instances [$__range]) by (node_id, go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} VUs" } - ], - "title": "VUs (Now)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 8, - "y": 1 - }, "id": 3, + "type": "stat", + "title": "Responses/sec (Now)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, - "textMode": "value_and_name", - "wideLayout": true + "textMode": "value_and_name" }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, + "refId": "A", + "queryType": "range", "expr": "sum(count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} [1s])) by (node_id, go_test_name, gen_name)", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Responses/sec", - "refId": "A" + "legendFormat": "{{go_test_name}} {{gen_name}} Responses/sec" } - ], - "title": "Responses/sec (Now)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 1 - }, "id": 4, + "type": "stat", + "title": "Successful requests (Total)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, - "textMode": "value_and_name", - "wideLayout": true + "textMode": "value_and_name" }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tsum(max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap success [$__range]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Successful requests", - "refId": "A" + "refId": "A", + "queryType": "instant", + "expr": "sum(count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"} !~ \"failed\\\":true\" [$__range])) by (node_id, go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} Successful requests" } - ], - "title": "Successful requests (Total)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 1 - }, "id": 5, + "type": "stat", + "title": "Errored requests (Total)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, "textMode": "value_and_name", - "wideLayout": true + "colorMode": "value" + }, + "fieldConfig": { + "defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}} }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tsum(max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap failed [$__range]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Errored requests", - "refId": "A" + "refId": "A", + "queryType": "instant", + "expr": "sum by (node_id, go_test_name, gen_name) (count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"} |~ \"failed\\\":true\" [$__range])) or sum by (node_id, go_test_name, gen_name) (count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"}[$__range])) * 0", + "legendFormat": "{{go_test_name}} {{gen_name}} Errored requests" } - ], - "title": "Errored requests (Total)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 1 - }, "id": 6, + "type": "stat", + "title": "Timed out requests (Total)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 1}, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12, - "valueSize": 20 - }, "textMode": "value_and_name", - "wideLayout": true + "colorMode": "value" + }, + "fieldConfig": { + "defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}} }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tsum(max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap callTimeout [$__range]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Timed out requests", - "refId": "A" + "refId": "A", + "queryType": "instant", + "expr": "sum by (node_id, go_test_name, gen_name) (count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"} |~ \"timeout\\\":true\" [$__range])) or sum by (node_id, go_test_name, gen_name) (count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"}[$__range])) * 0", + "legendFormat": "{{go_test_name}} {{gen_name}} Timed out requests" } - ], - "title": "Timed out requests (Total)", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "", - "spanNulls": false, - "stacking": { - "group": "", - "mode": "" - }, - "thresholdsStyle": { - "mode": "" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "height": "300px", - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, + "id": 10, + "type": "timeseries", + "title": "RPS/VUs per schedule segments", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 5}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tmax_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_rps [$__interval]) by (node_id, go_test_name, gen_name)\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} RPS", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_rps [$__interval]) by (node_id, go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} RPS" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tsum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_rps [$__interval]) by (node_id, go_test_name, gen_name))\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} Total RPS", - "refId": "B" + "refId": "B", + "queryType": "range", + "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_rps [$__interval]) by (node_id, go_test_name, gen_name))", + "legendFormat": "{{go_test_name}} Total RPS" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tmax_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_instances [$__interval]) by (node_id, go_test_name, gen_name)\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} VUs", - "refId": "C" + "refId": "C", + "queryType": "range", + "expr": "max_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_instances [$__interval]) by (node_id, go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} VUs" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\tsum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t| json\n\t\t\t| unwrap current_instances [$__interval]) by (node_id, go_test_name, gen_name))\n\t\t\t", - "format": "time_series", - "legendFormat": "{{go_test_name}} Total VUs", - "refId": "D" + "refId": "D", + "queryType": "range", + "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap current_instances [$__interval]) by (node_id, go_test_name, gen_name))", + "legendFormat": "{{go_test_name}} Total VUs" } - ], - "title": "RPS/VUs per schedule segments", - "transparent": true, - "type": "timeseries" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Responses", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "", - "spanNulls": false, - "stacking": { - "group": "", - "mode": "" - }, - "thresholdsStyle": { - "mode": "" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "Responses" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "height": "300px", - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, + "id": 11, + "type": "timeseries", + "title": "Responses/sec (Generator, CallGroup)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 5}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, + "refId": "A", + "queryType": "range", "expr": "sum(count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group:pipe}\"} [1s])) by (node_id, go_test_name, gen_name, call_group)", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} responses/sec", - "refId": "A" + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} responses/sec" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, + "refId": "B", + "queryType": "range", "expr": "sum(count_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} [1s])) by (node_id, go_test_name, gen_name)", - "format": "time_series", - "legendFormat": "{{go_test_name}} Total responses/sec", - "refId": "B" + "legendFormat": "{{go_test_name}} Total responses/sec" } - ], - "title": "Responses/sec (Generator, CallGroup)", - "transparent": true, - "type": "timeseries" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "ms", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "", - "spanNulls": false, - "stacking": { - "group": "", - "mode": "" - }, - "thresholdsStyle": { - "mode": "" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "height": "300px", - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, + "id": 12, + "type": "timeseries", + "title": "Latency quantiles over groups (99, 95, 50)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 13}, + "fieldConfig": {"defaults": {"unit": "ms"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\t\tquantile_over_time(0.99, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t\t| json\n\t\t\t\t| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Q 99 - {{error}}", - "refId": "A" + "refId": "Q99", + "queryType": "range", + "expr": "quantile_over_time(0.99, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", + "legendFormat": "{{go_test_name}} {{gen_name}} Q 99 - {{error}}" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\t\tquantile_over_time(0.95, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t\t| json\n\t\t\t\t| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Q 95 - {{error}}", - "refId": "B" + "refId": "Q95", + "queryType": "range", + "expr": "quantile_over_time(0.95, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", + "legendFormat": "{{go_test_name}} {{gen_name}} Q 95 - {{error}}" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\t\tquantile_over_time(0.50, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t\t| json\n\t\t\t\t| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} Q 50 - {{error}}", - "refId": "C" + "refId": "Q50", + "queryType": "range", + "expr": "quantile_over_time(0.50, {branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap duration [$__interval]) by (go_test_name, gen_name) / 1e6", + "legendFormat": "{{go_test_name}} {{gen_name}} Q 50 - {{error}}" } - ], - "title": "Latency quantiles over groups (99, 95, 50)", - "transparent": true, - "type": "timeseries" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "ms", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "", - "spanNulls": false, - "stacking": { - "group": "", - "mode": "" - }, - "thresholdsStyle": { - "mode": "" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "height": "300px", - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, + "id": 13, + "type": "timeseries", + "title": "Responses latencies by types over time (Generator, CallGroup)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 13}, + "fieldConfig": {"defaults": {"unit": "ms"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\t\tlast_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group}\"}\n\t\t\t\t| json\n\t\t\t\t| unwrap duration [$__interval]) / 1e6", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} T: {{timeout}} E: {{error}}", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\", call_group=~\"${call_group}\"}\n| json\n| unwrap duration [$__interval]) / 1e6", + "legendFormat": "{{go_test_name}} {{gen_name}} {{call_group}} T: {{timeout}} E: {{error}}" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n\t\t\t\tlast_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n\t\t\t\t| json\n\t\t\t\t| unwrap duration [$__interval]) / 1e6", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} all groups T: {{timeout}} E: {{error}}", - "refId": "B" + "refId": "B", + "queryType": "range", + "expr": "last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap duration [$__interval]) / 1e6", + "legendFormat": "{{go_test_name}} {{gen_name}} all groups T: {{timeout}} E: {{error}}" } - ], - "title": "Responses latencies by types over time (Generator, CallGroup)", - "transparent": true, - "type": "timeseries" + ] }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "P5DCFC7561CCDE821" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 17, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P5DCFC7561CCDE821" - }, - "refId": "A" - } - ], - "title": "WASP Load Test", - "type": "row" + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 21}, + "id": 200, + "title": "WASP Debug", + "type": "row", + "panels": [] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds", - "seriesBy": "last" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 25 - }, - "height": "100px", - "id": 11, + "id": 20, + "type": "stat", + "title": "Latest segment stats", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 6, "w": 24, "x": 0, "y": 22}, "options": { - "colorMode": "value", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "graphMode": "area", - "justifyMode": "", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, "textMode": "value_and_name", - "wideLayout": true + "colorMode": "value" }, - "pluginVersion": "10.2.6", - "renderer": "flot", "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n sum(bytes_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", gen_name=~\"${gen_name:pipe}\"} [$__range]) * 1e-6)\n ", - "format": "time_series", - "legendFormat": "Overall logs size", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "sum(bytes_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", gen_name=~\"${gen_name:pipe}\"} [$__range]) * 1e-6)", + "legendFormat": "Overall logs size" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n sum(bytes_rate({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", gen_name=~\"${gen_name:pipe}\"} [$__interval]) * 1e-6)\n ", - "format": "time_series", - "legendFormat": "Logs size per second", - "refId": "B" + "refId": "B", + "queryType": "range", + "expr": "sum(bytes_rate({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", gen_name=~\"${gen_name:pipe}\"} [$__interval]) * 1e-6)", + "legendFormat": "Logs size per second" } - ], - "title": "Latest segment stats", - "transparent": true, - "type": "stat" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "CallResults", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "", - "spanNulls": false, - "stacking": { - "group": "", - "mode": "" - }, - "thresholdsStyle": { - "mode": "" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 28 - }, - "height": "200px", - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, + "id": 21, + "type": "timeseries", + "title": "CallResult sampling (successful results)", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n | json\n | unwrap samples_recorded [$__interval])) by (go_test_name, gen_name)\n ", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} recorded", - "refId": "A" + "refId": "REC", + "queryType": "range", + "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap samples_recorded [$__interval])) by (go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} recorded" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "\n sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n | json\n | unwrap samples_skipped [$__interval])) by (go_test_name, gen_name)\n ", - "format": "time_series", - "legendFormat": "{{go_test_name}} {{gen_name}} skipped", - "refId": "B" + "refId": "SKIP", + "queryType": "range", + "expr": "sum(last_over_time({branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}\n| json\n| unwrap samples_skipped [$__interval])) by (go_test_name, gen_name)", + "legendFormat": "{{go_test_name}} {{gen_name}} skipped" } - ], - "title": "CallResult sampling (successful results)", - "transparent": true, - "type": "timeseries" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 34 - }, - "height": "300px", - "id": 13, - "options": { - "dedupStrategy": "", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, + "id": 30, + "type": "logs", + "title": "Stats logs", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 36}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"stats\", gen_name=~\"${gen_name:pipe}\"}" } - ], - "title": "Stats logs", - "transparent": true, - "type": "logs" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 42 - }, - "height": "300px", - "id": 14, - "options": { - "dedupStrategy": "", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, + "id": 31, + "type": "logs", + "title": "Failed responses", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} |~ \"failed\\\":true\"", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} |~ \"failed\\\":true\"" } - ], - "title": "Failed responses", - "transparent": true, - "type": "logs" + ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editable": false, - "error": false, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 42 - }, - "height": "300px", - "id": 15, - "options": { - "dedupStrategy": "", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, + "id": 32, + "type": "logs", + "title": "Timed out responses", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, "targets": [ { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} |~ \"timeout\\\":true\"", - "refId": "A" + "refId": "A", + "queryType": "range", + "expr": "{branch=~\"${branch:pipe}\", commit=~\"${commit:pipe}\", go_test_name=~\"${go_test_name:pipe}\", test_data_type=~\"responses\", gen_name=~\"${gen_name:pipe}\"} |~ \"timeout\\\":true\"" } - ], - "title": "Timed out responses", - "transparent": true, - "type": "logs" + ] } ], "refresh": "5s", "schemaVersion": 39, - "tags": [ - "generated", - "load-test" - ], + "tags": ["generated", "load-test", "wasp", "loki"], "templating": { "list": [ { - "allFormat": "", - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "go_test_name", - "multi": true, - "multiFormat": "", "name": "go_test_name", - "options": [], - "query": "label_values(go_test_name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "type": "query" + "label": "Go test name", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "query": {"label": "go_test_name", "type": 1, "refId": "LokiVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 3 }, { - "allFormat": "", - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "gen_name", - "multi": true, - "multiFormat": "", "name": "gen_name", - "options": [], - "query": "label_values(gen_name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "type": "query" + "label": "Generator name", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "query": {"label": "gen_name", "type": 1, "refId": "LokiVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 3 }, { - "allFormat": "", - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "branch", - "multi": true, - "multiFormat": "", "name": "branch", - "options": [], - "query": "label_values(branch)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "type": "query" + "label": "Branch", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "query": {"label": "branch", "type": 1, "refId": "LokiVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 3 }, { - "allFormat": "", - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "commit", - "multi": true, - "multiFormat": "", "name": "commit", - "options": [], - "query": "label_values(commit)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "type": "query" + "label": "Commit", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "query": {"label": "commit", "type": 1, "refId": "LokiVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 3 }, { - "allFormat": "", - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "call_group", - "multi": true, - "multiFormat": "", "name": "call_group", - "options": [], - "query": "label_values(call_group)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "type": "query" + "label": "Call group", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "query": {"label": "call_group", "type": 1, "refId": "LokiVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "multi": true, + "includeAll": true, + "sort": 3 } ] }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "WASP Load Test", - "uid": "WASPLoadTests", - "version": 11, + "time": {"from": "now-30m", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "WASP (Loki)", + "uid": "wasp-loki", + "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/framework/observability/compose/conf/provisioning/datasources/loki.yaml b/framework/observability/compose/conf/provisioning/datasources/loki.yaml index a176bc3bb..871cc58de 100644 --- a/framework/observability/compose/conf/provisioning/datasources/loki.yaml +++ b/framework/observability/compose/conf/provisioning/datasources/loki.yaml @@ -36,6 +36,17 @@ datasources: matcherRegex: '"traceID":"([a-f0-9]+)"' name: TraceID url: '$${__value.raw}' + # Alias of the Loki datasource above under uid "loki" so the embedded WASP + # dashboard (which references uid "loki", matching the WASP standalone stack) + # resolves without having to patch the dashboard JSON on every re-sync. + - name: Loki (WASP) + type: loki + uid: loki + isDefault: false + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 5000 - name: Prometheus type: prometheus uid: PBFA97CFB590B2093