feat(webapp): add per-worker Node.js heap metrics (#3437)

## Summary

Adds direct V8 heap and process-memory gauges to the webapp's
OpenTelemetry meter. The webapp already exports per-cluster-worker
Node.js runtime metrics (event-loop lag / utilization, active handles,
active requests, libuv threadpool size) via a custom meter under the
`trigger.dev` scope. Heap and memory were missing; this PR adds them
alongside, in the same observable-batch pattern.

## New gauges

| Metric | Source | Unit |
| --- | --- | --- |
| `nodejs.memory.heap.used` | `process.memoryUsage().heapUsed` | bytes |
| `nodejs.memory.heap.total` | `process.memoryUsage().heapTotal` | bytes
|
| `nodejs.memory.heap.limit` | `v8.getHeapStatistics().heap_size_limit`
| bytes |
| `nodejs.memory.external` | `process.memoryUsage().external` | bytes |
| `nodejs.memory.array_buffers` | `process.memoryUsage().arrayBuffers` |
bytes |
| `nodejs.memory.rss` | `process.memoryUsage().rss` | bytes |

Gated by the existing `INTERNAL_OTEL_NODEJS_METRICS_ENABLED` flag, same
as the adjacent event-loop / handle gauges. Zero overhead when disabled.

## Why

`@opentelemetry/host-metrics` publishes `process.memory.usage`, which is
RSS only. RSS is the sum of V8 heap, external memory (Buffers, etc.),
native code, and thread stacks. Without a direct heap metric it is not
possible to size the V8 heap cap (`--max-old-space-size`) from metrics
alone, because RSS overstates heap by the external + native footprint. A
worker can have a 4 GB RSS with a 2.5 GB heap and 1.5 GB of buffers; the
former constrains `--max-old-space-size`, the latter does not.

`nodejs.memory.heap.limit` also surfaces the configured
`--max-old-space-size` (read from
`v8.getHeapStatistics().heap_size_limit`), so operators can see the
current limit in the same dashboard as actual usage rather than
cross-referencing container environment variables.

## Risk

Minimal. Observable gauges are sampled at the configured metric-export
interval. `v8.getHeapStatistics()` and `process.memoryUsage()` are each
microsecond-level calls, and six gauges are added to the same batch
callback that already reads ~20 other Node.js runtime values per sample.
Same registration pattern as the existing event-loop metrics in the
file.

## Test plan

- [ ] Deploy and confirm the six new gauges appear at the configured
exporter
- [ ] In cluster mode, confirm per-worker granularity (one series per
cluster worker, tagged by `process.executable.name` /
`service.instance.id`)
- [ ] Confirm `nodejs.memory.heap.limit` reports the configured
`--max-old-space-size` value in bytes

Author: ericallam

.server-changes/nodejs-heap-metrics.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Add per-worker Node.js heap metrics to the OTel meter — `nodejs.memory.heap.used`, `nodejs.memory.heap.total`, `nodejs.memory.heap.limit`, `nodejs.memory.external`, `nodejs.memory.array_buffers`, `nodejs.memory.rss`. Host-metrics only publishes RSS, which overstates V8 heap by the external + native footprint; these give direct heap visibility per cluster worker so `NODE_MAX_OLD_SPACE_SIZE` can be sized against observed heap peaks rather than RSS.

apps/webapp/app/v3/tracer.server.ts

@@ -38,6 +38,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { PrismaInstrumentation } from "@prisma/instrumentation";
 import { HostMetrics } from "@opentelemetry/host-metrics";
 import { AwsInstrumentation as AwsSdkInstrumentation } from "@opentelemetry/instrumentation-aws-sdk";
+import v8 from "node:v8";
 import { awsEcsDetector, awsEc2Detector } from "@opentelemetry/resource-detector-aws";
 import {
   detectResources,
@@ -630,6 +631,39 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
     unit: "1", // OpenTelemetry convention for ratios
   });
 
+  // V8 heap + process memory. `NODE_MAX_OLD_SPACE_SIZE` caps V8 old space
+  // (reflected in `heap.limit`), but doesn't cap external/arrayBuffers/native
+  // memory — which is why RSS can exceed the heap total. Tracking all of these
+  // per-worker lets us size `NODE_MAX_OLD_SPACE_SIZE` against observed heap
+  // peaks rather than RSS (which overstates heap by the external + native
+  // footprint). `host-metrics` already publishes `process.memory.usage`
+  // (RSS), but we duplicate it under `nodejs.memory.rss` so all the memory
+  // numbers land in the same scope and are queryable together.
+  const heapUsedGauge = meter.createObservableGauge("nodejs.memory.heap.used", {
+    description: "V8 heap actively in use after the last GC",
+    unit: "By",
+  });
+  const heapTotalGauge = meter.createObservableGauge("nodejs.memory.heap.total", {
+    description: "V8 heap reserved (young + old generations)",
+    unit: "By",
+  });
+  const heapLimitGauge = meter.createObservableGauge("nodejs.memory.heap.limit", {
+    description: "V8 heap size limit (configured via --max-old-space-size)",
+    unit: "By",
+  });
+  const externalMemoryGauge = meter.createObservableGauge("nodejs.memory.external", {
+    description: "Memory used by C++ objects bound to JS (Buffer, etc.)",
+    unit: "By",
+  });
+  const arrayBuffersGauge = meter.createObservableGauge("nodejs.memory.array_buffers", {
+    description: "Memory allocated for ArrayBuffers and SharedArrayBuffers",
+    unit: "By",
+  });
+  const rssGauge = meter.createObservableGauge("nodejs.memory.rss", {
+    description: "Resident set size — total physical memory held by the process",
+    unit: "By",
+  });
+
   // Get UV threadpool size (defaults to 4 if not set)
   const uvThreadpoolSize = parseInt(process.env.UV_THREADPOOL_SIZE || "4", 10);
 
@@ -683,10 +717,16 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       currentEventLoopUtilization,
       lastEventLoopUtilization
     );
+    // Rotate the baseline so the next collection reports per-interval
+    // utilization rather than the cumulative average from process start.
+    lastEventLoopUtilization = currentEventLoopUtilization;
 
     // diff.utilization is between 0 and 1 (fraction of time "active")
     const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0;
 
+    const mem = process.memoryUsage();
+    const heapStats = v8.getHeapStatistics();
+
     return {
       threadpoolSize: uvThreadpoolSize,
       handlesByType,
@@ -702,6 +742,14 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
         p99: eventLoopLagP99?.values?.[0]?.value ?? 0,
         utilization,
       },
+      memory: {
+        heapUsed: mem.heapUsed,
+        heapTotal: mem.heapTotal,
+        heapLimit: heapStats.heap_size_limit,
+        external: mem.external,
+        arrayBuffers: mem.arrayBuffers,
+        rss: mem.rss,
+      },
     };
   }
 
@@ -714,6 +762,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
         requestsByType,
         requestsTotal,
         eventLoop,
+        memory,
       } = await readNodeMetrics();
 
       // Observe UV threadpool size
@@ -739,6 +788,14 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       res.observe(eventLoopLagP90Gauge, eventLoop.p90);
       res.observe(eventLoopLagP99Gauge, eventLoop.p99);
       res.observe(eluGauge, eventLoop.utilization);
+
+      // Observe memory metrics (bytes)
+      res.observe(heapUsedGauge, memory.heapUsed);
+      res.observe(heapTotalGauge, memory.heapTotal);
+      res.observe(heapLimitGauge, memory.heapLimit);
+      res.observe(externalMemoryGauge, memory.external);
+      res.observe(arrayBuffersGauge, memory.arrayBuffers);
+      res.observe(rssGauge, memory.rss);
     },
     [
       uvThreadpoolSizeGauge,
@@ -753,6 +810,12 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       eventLoopLagP90Gauge,
       eventLoopLagP99Gauge,
       eluGauge,
+      heapUsedGauge,
+      heapTotalGauge,
+      heapLimitGauge,
+      externalMemoryGauge,
+      arrayBuffersGauge,
+      rssGauge,
     ]
   );
 }