From 89f1f3b5b65df9be8d8e5630df5b5b5eb948e936 Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Wed, 3 Jun 2026 09:51:23 +0100 Subject: [PATCH 1/6] [DOC-14397]: DOC: Application Telemetry needs additional info. Added diagram and section for the histograms. Made corrections following the Copilot review Added more information on application telemetry to application-telemetry.adoc Added link to the new information on metrics-reference.adoc --- .../pages/application-telemetry-metrics.adoc | 120 ++++++++++++++++++ .../pages/metrics-reference.adoc | 2 + .../rest-api/pages/application-telemetry.adoc | 86 ++++++++++++- 3 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 modules/metrics-reference/pages/application-telemetry-metrics.adoc diff --git a/modules/metrics-reference/pages/application-telemetry-metrics.adoc b/modules/metrics-reference/pages/application-telemetry-metrics.adoc new file mode 100644 index 0000000000..dcc7c2af26 --- /dev/null +++ b/modules/metrics-reference/pages/application-telemetry-metrics.adoc @@ -0,0 +1,120 @@ += Application Telemetry Metrics +:description: Couchbase can be configured to collect metrics related to Couchbase SDK calls made from applications. +:stem: asciimath + +[abstract] +{description} + +== Prerequisites + +=== Activating Telemetry on Couchbase Server + +Before generating metrics from application telemetry, you must enable application telemetry on your Couchbase Server installation. +You can do this by executing the following CURL command: + +[source, bash, subs="+quotes"] +---- +curl -u Administrator:password -X POST \ +https://##{host}##:##{port}##/settings/appTelemetry \ +-d enabled=true +---- + +For more information on enabling application telemetry, see xref:rest-api:application-telemetry.adoc[] + +=== Activating Telemetry in your Application + +To activate telemetry recording, you will need to set the telemetry endpoint within your application. + +[source] +---- +couchbases://abc.cloud.couchbase.com?app_telemetry_endpoint=ws://localhost:9090/app_telemetry +---- + +Alternatively, you can set the endpoint using `ClusterOptions`. + +[source] +---- +ClusterOptions options; +options.appTelemetryEndpoint("ws://localhost:9090/app_telemetry"); +---- + +== Telemetry Counters + +Couchbase can record the number of SDK calls made for specific services. The metrics follow the same basic patterns: + +[%header] +|=== +| Metric| Description + +| `sdk_\{service}_r_total` +| The total number of operations + +|`sdk_\{service}_r_canceled` +| The total number of operations that were canceled. + +| `sdk_\{service}_r_timedout` +| The total number of operations that timed out. + +|=== + +To access the counter for a particular service, you replace `\{service}` with one of the following: + +* `kv` +* `query` +* `search` +* `analytics` +* `management` +* `eventing` + +So to access the number of calls that timed out for the query service, +you would use the command: `sdk_query_r_timedout`. + +== Histograms + +The same service labeling is used in the retrieval of histograms: + + +[%header, cols="2,1"] +|=== +| Metric| Description + +| `sdk_\{service}_retrieval_duration_milliseconds_bucket` +| Time take for retrieval + +|`sdk_\{service}_mutation_nondurable` +| Time taken for non-durable mutations. + +| `sdk_\{service}__mutation_durable` +| Time taken for ndurable mutations. +|=== + +The histogram metrics can take the same `service` values as the metric counters. + +* `kv` +* `query` +* `search` +* `analytics` +* `management` +* `eventing` + +[#histogram-output] +.Histogram output +==== +[source] +---- +sdk_kv_retrieval_duration_milliseconds_bucket{le="1",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 24054 +sdk_kv_retrieval_duration_milliseconds_bucket{le="10",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 33444 +sdk_kv_retrieval_duration_milliseconds_bucket{le="100",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 100392 +sdk_kv_retrieval_duration_milliseconds_bucket{le="500",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 129389 +sdk_kv_retrieval_duration_milliseconds_bucket{le="1000",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 133988 +sdk_kv_retrieval_duration_milliseconds_bucket{le="2500",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 139823 +sdk_kv_retrieval_duration_milliseconds_bucket{le="+Inf",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 144320 +sdk_kv_retrieval_duration_milliseconds_sum{agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 53423000 +sdk_kv_retrieval_duration_milliseconds_count{agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 144320 +---- +==== +xref:histogram-output[xrefstyle=short] demonstrates a series of `kv` retrieval times for the buckets in a node. +Each line gives the retrieval count for a particular range. +The retrieval duration counts are grouped by latency buckets: for example, requests completed in stem:[le 1] ms total 24,054, those in stem:[le 10] ms total 33,444, and so on. + +There are two special metrics associated with each histogram: `_sum` and `_count` that represent the overall sum of all buckets, and total number of the points in the histogram. diff --git a/modules/metrics-reference/pages/metrics-reference.adoc b/modules/metrics-reference/pages/metrics-reference.adoc index aba34af7bc..7a81e52478 100644 --- a/modules/metrics-reference/pages/metrics-reference.adoc +++ b/modules/metrics-reference/pages/metrics-reference.adoc @@ -26,3 +26,5 @@ In this section, metrics are listed according to service, as follows: * xref:metrics-reference:ns-server-metrics.adoc[Cluster Manager Metrics] * xref:metrics-reference:xdcr-metrics.adoc[XDCR Metrics] + +* xref:rest-api:application-telemetry.adoc[Application Telemetry Metrics] diff --git a/modules/rest-api/pages/application-telemetry.adoc b/modules/rest-api/pages/application-telemetry.adoc index b5194b9515..e418eac77b 100644 --- a/modules/rest-api/pages/application-telemetry.adoc +++ b/modules/rest-api/pages/application-telemetry.adoc @@ -11,8 +11,38 @@ Having Couchbase Server collect telemetry from your applications can help you troubleshoot client issues. This telemetry data is useful to diagnose issues such as poor performance or timeouts. -When you enable application telemetry, Couchbase Server advertises to SDK clients that it can collect telemetry data. -When an SDK client connects to a cluster with application telemetry enabled, it opens a WebSocket connection to a node in the cluster. +[mermaid] +.Telemetry Architecture +---- +flowchart LR +Client@{shape:rectange, label: "SDK Client"} + +subgraph Node1 [Couchbase Node] + direction LR + Server1@{shape:rectangle, label: "Couchbase Server"} + Prometheus1@{shape:rectangle, label: "Prometheus"} + Server1 -----> Prometheus1 +end + +subgraph Node2 [Couchbase Node] + direction LR + Server2@{shape:rectangle, label: "Couchbase Server"} + Prometheus2@{shape:rectangle, label: "Prometheus"} + Server2 --> Prometheus2 + +end + +Client --> Node1 +Client --> Node2 +Client -. websocket .-> Server2 +Server2 -. metrics forwarding .-> Server1 + +---- + + +When you enable application telemetry, Couchbase Server advertises to SDK clients that it can collect telemetry data. +When an SDK client connects to a cluster with application telemetry enabled, it opens a WebSocket connection to a node in the cluster. + +The clients will send metrics to any node in the cluster — the cluster will forward the metrics on to other nodes. + Couchbase Server uses this connection to periodically gather telemetry data from the client in Prometheus format. NOTE: Application telemetry is off by default in Couchbase Server 8.0. @@ -33,7 +63,6 @@ You cannot enable application telemetry if your cluster is running in mixed mode * Your applications must use a recent SDK version that supports application telemetry. The following table lists the SDKs that support application telemetry along with the version where they added support. - + |=== | SDK | Minimum Version with Application Telemetry Support @@ -73,6 +102,53 @@ The following table lists the SDKs that support application telemetry along with The default management port is 8091 for unencrypted connections and 18091 for encrypted connections. Make sure any firewall rules between your clients and the nodes allow traffic on the management port. +[IMPORTANT] +.Enabling telemetry +==== + +Telemetry is turned off by default on Couchbase Server. + +Use the following `curl` command to view the telemetry state on your server: + +[source, bash] +---- +curl -u Administrator:password -X GET http://localhost:8091/settings/appTelemetry | jq +---- + +This returns a JSON object that contains the telemetry state as `true` or `false`. + +[source, json] +---- +{ + "enabled": true, + "maxScrapeClientsPerNode": 1024, + "scrapeIntervalSeconds": 60 +} +---- + +Execute the following `curl` command to enable telemetry on your cluster. + +[source, bash] +---- +curl -u Administrator:password -X POST \ +http://localhost:8091/settings/appTelemetry \ +-d enabled=true +---- + +The SDKs have tracing enabled by default, but your applications should configure tracing options appropriately for the required telemetry information. + +More information about trace enabling can be found on the Java SDK pages: + +xref:java-sdk:howtos:observability-tracing.adoc[Observability Tracing] + +xref:java-sdk:howtos:observability-metrics.adoc[Observability Metrics] + +More examples can be found on the Couchbase GitHub pages: + +https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0084-app-service-level-telemetry.md[] + +==== + == HTTP Methods This API endpoint supports the following methods: @@ -251,7 +327,7 @@ This object has the same format as the <<#get-status-responses,response from the `400 Bad Request`:: Returned if you attempt to enable application telemetry on a cluster that's running in mixed mode where some nodes are running a version earlier than 8.0. -All of the nodes in the cluster must be running version 8.0 or later to enable application telemetry. +All the nodes in the cluster must be running version 8.0 or later to enable application telemetry. See <> for more requirements. `403 Forbidden`:: @@ -259,7 +335,7 @@ Returned if you do not have the proper roles to call this API. See <<#config-privs>> for a list of the required roles. `404 Not Found`:: -Returned if you attempt to call the endpoint on a running a version of Couchbase Server earlier than 8.0. +Returned if you attempt to call the endpoint on a version of Couchbase Server earlier than 8.0. [#config-examples] === Examples From 0b292780270bb81ca4732f518a57c95121c13344 Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Fri, 12 Jun 2026 12:39:21 +0100 Subject: [PATCH 2/6] [DOC-14397]: DOC: Application Telemetry needs additional info. Minor corrections Moved note on observability tracing to its own section. Signed-off-by: Ray Offiah --- .../pages/application-telemetry-metrics.adoc | 17 ----------- .../rest-api/pages/application-telemetry.adoc | 28 +++++++++++-------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/modules/metrics-reference/pages/application-telemetry-metrics.adoc b/modules/metrics-reference/pages/application-telemetry-metrics.adoc index dcc7c2af26..b1ec503410 100644 --- a/modules/metrics-reference/pages/application-telemetry-metrics.adoc +++ b/modules/metrics-reference/pages/application-telemetry-metrics.adoc @@ -21,23 +21,6 @@ https://##{host}##:##{port}##/settings/appTelemetry \ For more information on enabling application telemetry, see xref:rest-api:application-telemetry.adoc[] -=== Activating Telemetry in your Application - -To activate telemetry recording, you will need to set the telemetry endpoint within your application. - -[source] ----- -couchbases://abc.cloud.couchbase.com?app_telemetry_endpoint=ws://localhost:9090/app_telemetry ----- - -Alternatively, you can set the endpoint using `ClusterOptions`. - -[source] ----- -ClusterOptions options; -options.appTelemetryEndpoint("ws://localhost:9090/app_telemetry"); ----- - == Telemetry Counters Couchbase can record the number of SDK calls made for specific services. The metrics follow the same basic patterns: diff --git a/modules/rest-api/pages/application-telemetry.adoc b/modules/rest-api/pages/application-telemetry.adoc index e418eac77b..d1f798c6df 100644 --- a/modules/rest-api/pages/application-telemetry.adoc +++ b/modules/rest-api/pages/application-telemetry.adoc @@ -135,18 +135,6 @@ http://localhost:8091/settings/appTelemetry \ -d enabled=true ---- -The SDKs have tracing enabled by default, but your applications should configure tracing options appropriately for the required telemetry information. - -More information about trace enabling can be found on the Java SDK pages: - -xref:java-sdk:howtos:observability-tracing.adoc[Observability Tracing] - -xref:java-sdk:howtos:observability-metrics.adoc[Observability Metrics] - -More examples can be found on the Couchbase GitHub pages: - -https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0084-app-service-level-telemetry.md[] - ==== == HTTP Methods @@ -363,6 +351,22 @@ If successful, the previous command returns the following JSON object containing } ---- +[#observability-tracing] +== Observability Tracing + +The SDKs also have tracing enabled by default, but your applications should configure tracing options appropriately for the required telemetry information. + +More information about trace enabling can be found on the Java SDK pages: + +xref:java-sdk:howtos:observability-tracing.adoc[Observability Tracing] + +xref:java-sdk:howtos:observability-metrics.adoc[Observability Metrics] + +More examples can be found on the Couchbase GitHub pages: + +https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0084-app-service-level-telemetry.md[] + + == See Also * xref:manage:monitor/set-up-prometheus-for-monitoring.adoc[] From 7258a5cfde0d43e9fcfb2aea4334a4180c9b0182 Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Wed, 17 Jun 2026 10:28:54 +0100 Subject: [PATCH 3/6] [DOC-14397]: DOC: Application Telemetry needs additional info Signed-off-by: Ray Offiah --- modules/rest-api/pages/application-telemetry.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/rest-api/pages/application-telemetry.adoc b/modules/rest-api/pages/application-telemetry.adoc index d1f798c6df..d27fa30189 100644 --- a/modules/rest-api/pages/application-telemetry.adoc +++ b/modules/rest-api/pages/application-telemetry.adoc @@ -110,9 +110,9 @@ Telemetry is turned off by default on Couchbase Server. Use the following `curl` command to view the telemetry state on your server: -[source, bash] +[source, bash, subs="+quotes"] ---- -curl -u Administrator:password -X GET http://localhost:8091/settings/appTelemetry | jq +curl -u Administrator:password -X GET http://##{host}##:##{port}##/settings/appTelemetry | jq ---- This returns a JSON object that contains the telemetry state as `true` or `false`. @@ -128,10 +128,10 @@ This returns a JSON object that contains the telemetry state as `true` or `false Execute the following `curl` command to enable telemetry on your cluster. -[source, bash] +[source, bash, subs="+quotes"] ---- curl -u Administrator:password -X POST \ -http://localhost:8091/settings/appTelemetry \ +http://##{host}##:##{port}##/settings/appTelemetry \ -d enabled=true ---- From 3234ce433e881af162510a4c2f2a489607c3511f Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Wed, 17 Jun 2026 10:35:34 +0100 Subject: [PATCH 4/6] [DOC-14397]: DOC: Application Telemetry needs additional info Making examples consistent; using `localhost`. Signed-off-by: Ray Offiah --- modules/rest-api/pages/application-telemetry.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/rest-api/pages/application-telemetry.adoc b/modules/rest-api/pages/application-telemetry.adoc index d27fa30189..30eeef2cd3 100644 --- a/modules/rest-api/pages/application-telemetry.adoc +++ b/modules/rest-api/pages/application-telemetry.adoc @@ -110,9 +110,9 @@ Telemetry is turned off by default on Couchbase Server. Use the following `curl` command to view the telemetry state on your server: -[source, bash, subs="+quotes"] +[source, bash] ---- -curl -u Administrator:password -X GET http://##{host}##:##{port}##/settings/appTelemetry | jq +curl -u Administrator:password -X GET http://localhost:8091/settings/appTelemetry | jq ---- This returns a JSON object that contains the telemetry state as `true` or `false`. @@ -131,7 +131,7 @@ Execute the following `curl` command to enable telemetry on your cluster. [source, bash, subs="+quotes"] ---- curl -u Administrator:password -X POST \ -http://##{host}##:##{port}##/settings/appTelemetry \ +http://localhoast:8091/settings/appTelemetry \ -d enabled=true ---- From bfb9f483568d44c34dfbe3973621e99bbf475cac Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Wed, 1 Jul 2026 10:53:52 +0100 Subject: [PATCH 5/6] [DOC-14397]: DOC: Application Telemetry needs additional info. Corrections following review. Signed-off-by: Ray Offiah --- .../pages/application-telemetry-metrics.adoc | 107 +++++++++++------- .../rest-api/pages/application-telemetry.adoc | 28 +---- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/modules/metrics-reference/pages/application-telemetry-metrics.adoc b/modules/metrics-reference/pages/application-telemetry-metrics.adoc index b1ec503410..d1a226bfcc 100644 --- a/modules/metrics-reference/pages/application-telemetry-metrics.adoc +++ b/modules/metrics-reference/pages/application-telemetry-metrics.adoc @@ -1,29 +1,34 @@ = Application Telemetry Metrics -:description: Couchbase can be configured to collect metrics related to Couchbase SDK calls made from applications. +:description: Couchbase can be configured to collect metrics related to Couchbase SDK calls made from applications. :stem: asciimath +:xrefstyle: short [abstract] {description} -== Prerequisites +== Overview + +Couchbase Server can be configured to collect metrics related to Couchbase SDK calls made from applications. +The client SDKs send metrics to the Server that are then handled like regular server-side metrics, and you can access them through the Prometheus endpoint and the Stats API. -=== Activating Telemetry on Couchbase Server +== Prerequisites -Before generating metrics from application telemetry, you must enable application telemetry on your Couchbase Server installation. -You can do this by executing the following CURL command: +You must enable application telemetry collection on your Couchbase Server installation. +You can do this by executing the following curl command: [source, bash, subs="+quotes"] ---- curl -u Administrator:password -X POST \ -https://##{host}##:##{port}##/settings/appTelemetry \ +http[s]://##{host}##:##{port}##/settings/appTelemetry \ -d enabled=true ---- -For more information on enabling application telemetry, see xref:rest-api:application-telemetry.adoc[] +For more information about enabling application telemetry, see xref:rest-api:application-telemetry.adoc[] == Telemetry Counters -Couchbase can record the number of SDK calls made for specific services. The metrics follow the same basic patterns: +Couchbase can record the number of SDK calls made for specific services. +The metrics follow the same basic patterns: [%header] |=== @@ -49,55 +54,69 @@ To access the counter for a particular service, you replace `\{service}` with on * `management` * `eventing` -So to access the number of calls that timed out for the query service, +To access the number of calls that timed out for the Query Service, you would use the command: `sdk_query_r_timedout`. == Histograms +<> below shows the available application telemetry metrics. +The `_bucket` at the end of the metric names is a special suffix that signifies that the line should be interpreted as a bucket of that histogram. +Also, there are 2 special metrics associated with each histogram: `_sum` and `_count` that represent the overall sum of all histogram buckets, +and the total number of the points in the histogram. +The `_sum` value should be a total sum of the points in milliseconds represented as an integer. + +[#available-histograms] +=== Available Histograms + +* `sdk_kv_retrieval_duration_milliseconds_bucket` +* `sdk_kv_retrieval_duration_milliseconds_count` +* `sdk_kv_retrieval_duration_milliseconds_sum` +* `sdk_kv_mutation_nondurable_duration_milliseconds_bucket` +* `sdk_kv_mutation_nondurable_duration_milliseconds_count` +* `sdk_kv_mutation_nondurable_duration_milliseconds_sum` +* `sdk_kv_mutation_durable_duration_milliseconds_bucket` +* `sdk_kv_mutation_durable_duration_milliseconds_count` +* `sdk_kv_mutation_durable_duration_milliseconds_sum` +* `sdk_query_duration_milliseconds_bucket` +* `sdk_query_duration_milliseconds_count` +* `sdk_query_duration_milliseconds_sum` +* `sdk_search_duration_milliseconds_bucket` +* `sdk_query_duration_milliseconds_count` +* `sdk_query_duration_milliseconds_sum` +* `sdk_search_duration_milliseconds_bucket` +* `sdk_search_duration_milliseconds_count` +* `sdk_search_duration_milliseconds_sum` +* `sdk_analytics_duration_milliseconds_bucket` +* `sdk_analytics_duration_milliseconds_count` +* `sdk_analytics_duration_milliseconds_sum` +* `sdk_eventing_duration_milliseconds_bucket` +* `sdk_eventing_duration_milliseconds_count` +* `sdk_eventing_duration_milliseconds_sum` +* `sdk_management_duration_milliseconds_bucket` +* `sdk_management_duration_milliseconds_count` +* `sdk_management_duration_milliseconds_sum` -The same service labeling is used in the retrieval of histograms: - - -[%header, cols="2,1"] -|=== -| Metric| Description - -| `sdk_\{service}_retrieval_duration_milliseconds_bucket` -| Time take for retrieval - -|`sdk_\{service}_mutation_nondurable` -| Time taken for non-durable mutations. -| `sdk_\{service}__mutation_durable` -| Time taken for ndurable mutations. -|=== - -The histogram metrics can take the same `service` values as the metric counters. - -* `kv` -* `query` -* `search` -* `analytics` -* `management` -* `eventing` [#histogram-output] .Histogram output ==== [source] ---- -sdk_kv_retrieval_duration_milliseconds_bucket{le="1",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 24054 -sdk_kv_retrieval_duration_milliseconds_bucket{le="10",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 33444 -sdk_kv_retrieval_duration_milliseconds_bucket{le="100",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 100392 -sdk_kv_retrieval_duration_milliseconds_bucket{le="500",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 129389 -sdk_kv_retrieval_duration_milliseconds_bucket{le="1000",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 133988 -sdk_kv_retrieval_duration_milliseconds_bucket{le="2500",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 139823 -sdk_kv_retrieval_duration_milliseconds_bucket{le="+Inf",agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 144320 -sdk_kv_retrieval_duration_milliseconds_sum{agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 53423000 -sdk_kv_retrieval_duration_milliseconds_count{agent="sdk/2.4.5.0",bucket="travel-sample",node="node1",node_uuid="91442eb8202e0e16bbb59624d9ccdb0a"} 144320 +sdk_kv_retrieval_duration_milliseconds_bucket{le="1"} 24054 +sdk_kv_retrieval_duration_milliseconds_bucket{le="10"} 33444 +sdk_kv_retrieval_duration_milliseconds_bucket{le="100"} 100392 +sdk_kv_retrieval_duration_milliseconds_bucket{le="500"} 129389 +sdk_kv_retrieval_duration_milliseconds_bucket{le="1000"} 133988 +sdk_kv_retrieval_duration_milliseconds_bucket{le="2500"} 139823 +sdk_kv_retrieval_duration_milliseconds_bucket{le="+Inf"} 144320 +sdk_kv_retrieval_duration_milliseconds_sum{} 53423000 +sdk_kv_retrieval_duration_milliseconds_count{} 144320 ---- ==== xref:histogram-output[xrefstyle=short] demonstrates a series of `kv` retrieval times for the buckets in a node. Each line gives the retrieval count for a particular range. -The retrieval duration counts are grouped by latency buckets: for example, requests completed in stem:[le 1] ms total 24,054, those in stem:[le 10] ms total 33,444, and so on. +For example, the `le="1"` label is the upper bound of the latency bucket, so that range is stem:[le 1 ms]. +The value would fall into all buckets where the upper bound is higher. +As shown for the retrieval duration values, the requests completed in stem:[le 1] ms total 24,054, those in stem:[le 10] ms total 33,444 and so on. -There are two special metrics associated with each histogram: `_sum` and `_count` that represent the overall sum of all buckets, and total number of the points in the histogram. +There are 2 special metrics associated with each histogram: `_sum` and `_count` that represent the overall sum of all buckets, and total number of the points in the histogram. diff --git a/modules/rest-api/pages/application-telemetry.adoc b/modules/rest-api/pages/application-telemetry.adoc index 30eeef2cd3..287590df26 100644 --- a/modules/rest-api/pages/application-telemetry.adoc +++ b/modules/rest-api/pages/application-telemetry.adoc @@ -9,7 +9,7 @@ == Description Having Couchbase Server collect telemetry from your applications can help you troubleshoot client issues. -This telemetry data is useful to diagnose issues such as poor performance or timeouts. +This visibility into metrics data from the application's point of view is useful to diagnose issues such as poor performance or timeouts. [mermaid] .Telemetry Architecture @@ -19,14 +19,14 @@ Client@{shape:rectange, label: "SDK Client"} subgraph Node1 [Couchbase Node] direction LR - Server1@{shape:rectangle, label: "Couchbase Server"} + Server1@{shape:rectangle, label: "Couchbase Server Cluster Manager"} Prometheus1@{shape:rectangle, label: "Prometheus"} Server1 -----> Prometheus1 end subgraph Node2 [Couchbase Node] direction LR - Server2@{shape:rectangle, label: "Couchbase Server"} + Server2@{shape:rectangle, label: "Couchbase Server Cluster Manager"} Prometheus2@{shape:rectangle, label: "Prometheus"} Server2 --> Prometheus2 @@ -42,7 +42,7 @@ Server2 -. metrics forwarding .-> Server1 When you enable application telemetry, Couchbase Server advertises to SDK clients that it can collect telemetry data. When an SDK client connects to a cluster with application telemetry enabled, it opens a WebSocket connection to a node in the cluster. + -The clients will send metrics to any node in the cluster — the cluster will forward the metrics on to other nodes. + +The clients send metrics to any node in the cluster — the cluster forwards the metrics on to other nodes. + Couchbase Server uses this connection to periodically gather telemetry data from the client in Prometheus format. NOTE: Application telemetry is off by default in Couchbase Server 8.0. @@ -120,7 +120,7 @@ This returns a JSON object that contains the telemetry state as `true` or `false [source, json] ---- { - "enabled": true, + "enabled": false, "maxScrapeClientsPerNode": 1024, "scrapeIntervalSeconds": 60 } @@ -292,7 +292,7 @@ The default value is `60`. + You can increase this value to reduce the overhead of collecting telemetry data on your nodes. -However, increasing this value means Couchbase Server will miss collecting more telemetry data from clients before they disconnect. +However, increasing this value means Couchbase Server misses collecting more telemetry data from clients before they disconnect. For example, suppose you set this value to `300`. Then Couchbase Server could lose up to 5 minutes of telemetry data from a client that disconnects just before the next scheduled telemetry collection. @@ -351,22 +351,6 @@ If successful, the previous command returns the following JSON object containing } ---- -[#observability-tracing] -== Observability Tracing - -The SDKs also have tracing enabled by default, but your applications should configure tracing options appropriately for the required telemetry information. - -More information about trace enabling can be found on the Java SDK pages: - -xref:java-sdk:howtos:observability-tracing.adoc[Observability Tracing] - -xref:java-sdk:howtos:observability-metrics.adoc[Observability Metrics] - -More examples can be found on the Couchbase GitHub pages: - -https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0084-app-service-level-telemetry.md[] - - == See Also * xref:manage:monitor/set-up-prometheus-for-monitoring.adoc[] From c869e82ae3b2b1af22f3c38845eb066b08b1cea6 Mon Sep 17 00:00:00 2001 From: Ray Offiah Date: Fri, 3 Jul 2026 09:13:09 +0100 Subject: [PATCH 6/6] [DOC-14397]: DOC: Application Telemetry needs additional info. Removed duplicated lines from histogram list. Signed-off-by: Ray Offiah --- .../metrics-reference/pages/application-telemetry-metrics.adoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/metrics-reference/pages/application-telemetry-metrics.adoc b/modules/metrics-reference/pages/application-telemetry-metrics.adoc index d1a226bfcc..d89db10d1c 100644 --- a/modules/metrics-reference/pages/application-telemetry-metrics.adoc +++ b/modules/metrics-reference/pages/application-telemetry-metrics.adoc @@ -80,9 +80,6 @@ The `_sum` value should be a total sum of the points in milliseconds represented * `sdk_query_duration_milliseconds_count` * `sdk_query_duration_milliseconds_sum` * `sdk_search_duration_milliseconds_bucket` -* `sdk_query_duration_milliseconds_count` -* `sdk_query_duration_milliseconds_sum` -* `sdk_search_duration_milliseconds_bucket` * `sdk_search_duration_milliseconds_count` * `sdk_search_duration_milliseconds_sum` * `sdk_analytics_duration_milliseconds_bucket`