feat(metrics): Add OTEL to api

apps/api/package.json

@@ -11,14 +11,14 @@
   "scripts": {
     "build": "tsc && tsc-alias",
     "clean": "rm -rf .turbo node_modules",
-    "dev": "pnpm with-env tsx watch --clear-screen=false src/index.ts",
+    "dev": "pnpm with-env tsx watch --clear-screen=false --import ./src/instrumentation.ts src/index.ts",
     "typecheck": "tsc --noEmit --emitDeclarationOnly false",
     "format": "prettier --check . --ignore-path ../../.gitignore --ignore-path ./src/types/openapi.ts --ignore-path ./openapi/openapi.json && bash -c 'jsonnetfmt -i ./**/*.jsonnet'",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore --ignore-path ./src/types/openapi.ts --ignore-path ./openapi/openapi.json && bash -c 'jsonnetfmt -i ./**/*.jsonnet'",
     "generate": "jsonnet openapi/main.jsonnet > openapi/openapi.json && pnpm generate:types",
     "generate:types": "openapi-typescript openapi/openapi.json -o src/types/openapi.ts",
     "lint": "eslint",
-    "start": "node dist/index.js",
+    "start": "node --import ./dist/src/instrumentation.js dist/src/index.js",
     "with-env": "dotenv -e ../../.env --"
   },
   "dependencies": {
@@ -33,6 +33,18 @@
     "@octokit/rest": "catalog:",
     "@octokit/webhooks": "^13.7.4",
     "@octokit/webhooks-types": "^7.5.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.217.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.217.0",
+    "@opentelemetry/instrumentation-express": "^0.65.0",
+    "@opentelemetry/instrumentation-http": "^0.217.0",
+    "@opentelemetry/instrumentation-pg": "^0.69.0",
+    "@opentelemetry/instrumentation-runtime-node": "^0.30.0",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-metrics": "^2.7.1",
+    "@opentelemetry/sdk-node": "^0.217.0",
+    "@opentelemetry/sdk-trace-node": "^2.7.1",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
     "@t3-oss/env-core": "catalog:",
     "@trpc/server": "11.0.0-rc.364",
     "bcryptjs": "^2.4.3",

apps/api/src/config.ts

@@ -36,6 +36,10 @@ export const env = createEnv({
 
     BASE_URL: z.string().optional(),
 
+    OTEL_SERVICE_NAME: z.string().default("ctrlplane/api"),
+    OTEL_EXPORTER_OTLP_ENDPOINT: z
+      .string()
+      .default("http://localhost:4318"),
     OTEL_SAMPLER_RATIO: z.number().optional().default(1),
 
     AZURE_APP_CLIENT_ID: z.string().optional(),

apps/api/src/instrumentation.ts

@@ -0,0 +1,66 @@
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import {
+  ExpressInstrumentation,
+  ExpressLayerType,
+} from "@opentelemetry/instrumentation-express";
+import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
+import { PgInstrumentation } from "@opentelemetry/instrumentation-pg";
+import { RuntimeNodeInstrumentation } from "@opentelemetry/instrumentation-runtime-node";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import {
+  ParentBasedSampler,
+  TraceIdRatioBasedSampler,
+} from "@opentelemetry/sdk-trace-node";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+
+import { env } from "@/config.js";
+
+const sdk = new NodeSDK({
+  resource: resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: env.OTEL_SERVICE_NAME,
+  }),
+  sampler: new ParentBasedSampler({
+    root: new TraceIdRatioBasedSampler(env.OTEL_SAMPLER_RATIO),
+  }),
+  traceExporter: new OTLPTraceExporter(),
+  metricReader: new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter(),
+    exportIntervalMillis: 10_000,
+  }),
+  instrumentations: [
+    new HttpInstrumentation({
+      ignoreIncomingRequestHook: (req) => req.url === "/api/healthz",
+    }),
+    new ExpressInstrumentation({
+      ignoreLayersType: [ExpressLayerType.MIDDLEWARE],
+    }),
+    new PgInstrumentation(),
+    new RuntimeNodeInstrumentation(),
+  ],
+});
+
+try {
+  sdk.start();
+  console.log("OpenTelemetry started for service: ", env.OTEL_SERVICE_NAME);
+} catch (err) {
+  console.error(
+    "OpenTelemetry failed to start, continuing without telemetry",
+    err,
+  );
+}
+
+const shutdown = async () => {
+  try {
+    await sdk.shutdown();
+  } catch (err) {
+    console.error("OpenTelemetry shutdown failed", err);
+  } finally {
+    process.exit(0);
+  }
+};
+
+process.on("SIGTERM", () => void shutdown());
+process.on("SIGINT", () => void shutdown());

docs/installation.mdx

@@ -333,7 +333,7 @@ GITHUB_CLIENT_SECRET=your-client-secret
 # Logging
 LOG_LEVEL=info
 
-# OpenTelemetry (optional)
+# OpenTelemetry (optional) - To fully disable OTEL, set OTEL_SDK_DISABLED=true
 OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318
 ```
 

packages/trpc/package.json

@@ -28,6 +28,7 @@
     "@octokit/auth-app": "catalog:",
     "@octokit/rest": "catalog:",
     "@octokit/types": "^13.5.0",
+    "@opentelemetry/api": "^1.9.0",
     "@t3-oss/env-core": "catalog:",
     "@trpc/server": "11.0.0-rc.364",
     "better-auth": "^1.4.6",

packages/trpc/src/trpc.ts

@@ -1,5 +1,6 @@
 import type { Session } from "@ctrlplane/auth/server";
 import type { PermissionChecker } from "@ctrlplane/auth/utils";
+import { SpanStatusCode, trace } from "@opentelemetry/api";
 import { initTRPC, TRPCError } from "@trpc/server";
 import superjson from "superjson";
 import { ZodError } from "zod/v4";
@@ -43,7 +44,27 @@ const t = initTRPC
   });
 
 export const router = t.router;
-export const publicProcedure = t.procedure;
+
+const tracer = trace.getTracer("@ctrlplane/trpc");
+
+const tracingMiddleware = t.middleware(({ path, type, next }) =>
+  tracer.startActiveSpan(`trpc.${type} ${path}`, async (span) => {
+    span.setAttribute("trpc.path", path);
+    span.setAttribute("trpc.type", type);
+    try {
+      const result = await next();
+      if (!result.ok) {
+        span.setStatus({ code: SpanStatusCode.ERROR });
+        span.setAttribute("trpc.error_code", result.error.code);
+      }
+      return result;
+    } finally {
+      span.end();
+    }
+  }),
+);
+
+export const publicProcedure = t.procedure.use(tracingMiddleware);
 
 const authnProcedure = publicProcedure.use(({ ctx, next }) => {
   if (ctx.session == null) throw new TRPCError({ code: "UNAUTHORIZED" });