first commit

libraries/metrics/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+
+.npmrc

libraries/metrics/.npmignore

@@ -0,0 +1,4 @@
+/.circleci
+/.eslintrc
+/.nvmrc
+/.prettierrc

libraries/metrics/.nvmrc

@@ -0,0 +1 @@
+20.18.2

libraries/metrics/CHANGELOG.md

@@ -0,0 +1,9 @@
+## v4.1.0
+
+* Allows skipping the `sampleRate` argument.
+
+## v4.0.0
+
+* Send unmodified request and response to logger.
+
+This version of the metrics module only works with versions of the `@overleaf/logger` module greater than v3.0.0

libraries/metrics/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Overleaf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

libraries/metrics/README.md

@@ -0,0 +1,33 @@
+# overleaf/metrics-module
+
+Wrappers the [prom-client](https://github.com/siimon/prom-client) npm module to provide [Prometheus](https://prometheus.io/) metrics at `/metrics`.
+
+Use:
+
+```
+// Metrics must be initialized before importing anything else
+require('@overleaf/metrics/initialize')
+
+const express = require('express')
+const metrics = require('@overleaf/metrics')
+const app = express()
+metrics.injectMetricsRoute(app)
+```
+
+Request logging can be enabled:
+
+```
+const logger = require('@overleaf/logger')
+...
+app.use(metrics.http.monitor(logger))
+```
+
+The metrics module can be configured through the following environment variables:
+
+- `DEBUG_METRICS` - enables display of debugging messages to the console.
+- `GCP_OPENTELEMETRY` - enables OpenTelemetry tracing for GCP
+- `JAEGER_OPENTELEMETRY` - enables OpenTelemetry tracing for Jaeger (in the dev environment)
+- `METRICS_APP_NAME` - the app label for metrics and spans
+- `METRICS_COMPRESSION_LEVEL` - sets the [compression level](https://www.npmjs.com/package/compression#level) for `/metrics`
+- `STACKDRIVER_LOGGING` - toggles the request logging format
+- `UV_THREADPOOL_SIZE` - sets the libuv [thread pool](http://docs.libuv.org/en/v1.x/threadpool.html) size

libraries/metrics/buildscript.txt

@@ -0,0 +1,10 @@
+metrics
+--dependencies=None
+--docker-repos=gcr.io/overleaf-ops
+--env-add=
+--env-pass-through=
+--esmock-loader=False
+--is-library=True
+--node-version=20.18.2
+--public-repo=False
+--script-version=4.7.0

libraries/metrics/event_loop.js

@@ -0,0 +1,34 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+module.exports = {
+  monitor(logger, interval, logThreshold) {
+    if (interval == null) {
+      interval = 1000
+    }
+    if (logThreshold == null) {
+      logThreshold = 100
+    }
+    const Metrics = require('./index')
+    // check for logger on startup to avoid exceptions later if undefined
+    if (logger == null) {
+      throw new Error('logger is undefined')
+    }
+    // monitor delay in setInterval to detect event loop blocking
+    let previous = Date.now()
+    const intervalId = setInterval(function () {
+      const now = Date.now()
+      const offset = now - previous - interval
+      if (offset > logThreshold) {
+        logger.warn({ offset }, 'slow event loop')
+      }
+      previous = now
+      return Metrics.timing('event-loop-millsec', offset)
+    }, interval)
+
+    return Metrics.registerDestructor(() => clearInterval(intervalId))
+  },
+}

libraries/metrics/http.js

@@ -0,0 +1,78 @@
+const Metrics = require('./index')
+
+function monitor(logger, level = 'debug') {
+  return function (req, res, next) {
+    const startTime = Date.now()
+    req.logger = new RequestLogger(logger, level)
+    const { end } = res
+    res.end = function (...args) {
+      end.apply(this, args)
+      const responseTimeMs = Date.now() - startTime
+      const requestSize = parseInt(req.headers['content-length'], 10)
+      const routePath = getRoutePath(req)
+
+      if (routePath != null) {
+        Metrics.timing('http_request', responseTimeMs, null, {
+          method: req.method,
+          status_code: res.statusCode,
+          path: routePath,
+        })
+        if (requestSize) {
+          Metrics.summary('http_request_size_bytes', requestSize, {
+            method: req.method,
+            status_code: res.statusCode,
+            path: routePath,
+          })
+        }
+      }
+      req.logger.addFields({ responseTimeMs })
+      req.logger.emit(req, res)
+    }
+    next()
+  }
+}
+
+function getRoutePath(req) {
+  if (req.route && req.route.path != null) {
+    return req.route.path
+      .toString()
+      .replace(/\//g, '_')
+      .replace(/:/g, '')
+      .slice(1)
+  }
+  if (req.swagger && req.swagger.apiPath != null) {
+    return req.swagger.apiPath
+  }
+  return null
+}
+
+class RequestLogger {
+  constructor(logger, level) {
+    this._logger = logger
+    this._level = level
+    this._info = {}
+  }
+
+  addFields(fields) {
+    Object.assign(this._info, fields)
+  }
+
+  setLevel(level) {
+    this._level = level
+  }
+
+  disable() {
+    this._disabled = true
+  }
+
+  emit(req, res) {
+    if (this._disabled) {
+      return
+    }
+    this.addFields({ req, res })
+    const url = req.originalUrl || req.url
+    this._logger[this._level](this._info, '%s %s', req.method, url)
+  }
+}
+
+module.exports.monitor = monitor

libraries/metrics/index.js

@@ -0,0 +1,182 @@
+/* eslint-disable no-console */
+
+const ExpressCompression = require('compression')
+const promClient = require('prom-client')
+const promWrapper = require('./prom_wrapper')
+
+const destructors = []
+
+require('./uv_threadpool_size')
+
+function registerDestructor(func) {
+  destructors.push(func)
+}
+
+function injectMetricsRoute(app) {
+  app.get(
+    '/metrics',
+    ExpressCompression({
+      level: parseInt(process.env.METRICS_COMPRESSION_LEVEL || '1', 10),
+    }),
+    function (req, res, next) {
+      res.set('Content-Type', promWrapper.registry.contentType)
+      promWrapper.registry
+        .metrics()
+        .then(metrics => {
+          res.end(metrics)
+        })
+        .catch(err => {
+          next(err)
+        })
+    }
+  )
+}
+
+function buildPromKey(key) {
+  return key.replace(/[^a-zA-Z0-9]/g, '_')
+}
+
+function sanitizeValue(value) {
+  return parseFloat(value)
+}
+
+function set(key, value, sampleRate = 1) {
+  console.log('counts are not currently supported')
+}
+
+function inc(key, sampleRate = 1, labels = {}) {
+  if (arguments.length === 2 && typeof sampleRate === 'object') {
+    labels = sampleRate
+  }
+
+  key = buildPromKey(key)
+  promWrapper.metric('counter', key, labels).inc(labels)
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing inc', key, labels)
+  }
+}
+
+function count(key, count, sampleRate = 1, labels = {}) {
+  if (arguments.length === 3 && typeof sampleRate === 'object') {
+    labels = sampleRate
+  }
+
+  key = buildPromKey(key)
+  promWrapper.metric('counter', key, labels).inc(labels, count)
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing count/inc', key, labels)
+  }
+}
+
+function summary(key, value, labels = {}) {
+  key = buildPromKey(key)
+  promWrapper.metric('summary', key, labels).observe(labels, value)
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing summary', key, value, labels)
+  }
+}
+
+function timing(key, timeSpan, sampleRate = 1, labels = {}) {
+  if (arguments.length === 3 && typeof sampleRate === 'object') {
+    labels = sampleRate
+  }
+
+  key = buildPromKey('timer_' + key)
+  promWrapper.metric('summary', key, labels).observe(labels, timeSpan)
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing timing', key, labels)
+  }
+}
+
+function histogram(key, value, buckets, labels = {}) {
+  key = buildPromKey('histogram_' + key)
+  promWrapper.metric('histogram', key, labels, buckets).observe(labels, value)
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing histogram', key, buckets, labels)
+  }
+}
+
+class Timer {
+  constructor(key, sampleRate = 1, labels = {}, buckets = undefined) {
+    if (typeof sampleRate === 'object') {
+      // called with (key, labels, buckets)
+      if (arguments.length === 3) {
+        buckets = labels
+        labels = sampleRate
+      }
+
+      // called with (key, labels)
+      if (arguments.length === 2) {
+        labels = sampleRate
+      }
+
+      sampleRate = 1 // default value to pass to timing function
+    }
+
+    this.start = new Date()
+    key = buildPromKey(key)
+    this.key = key
+    this.sampleRate = sampleRate
+    this.labels = labels
+    this.buckets = buckets
+  }
+
+  // any labels passed into the done method override labels from constructor
+  done(labels = {}) {
+    const timeSpan = new Date() - this.start
+    if (this.buckets) {
+      histogram(this.key, timeSpan, this.buckets, { ...this.labels, ...labels })
+    } else {
+      timing(this.key, timeSpan, this.sampleRate, { ...this.labels, ...labels })
+    }
+    return timeSpan
+  }
+}
+
+function gauge(key, value, sampleRate = 1, labels = {}) {
+  if (arguments.length === 3 && typeof sampleRate === 'object') {
+    labels = sampleRate
+  }
+
+  key = buildPromKey(key)
+  promWrapper.metric('gauge', key, labels).set(labels, sanitizeValue(value))
+  if (process.env.DEBUG_METRICS) {
+    console.log('doing gauge', key, labels)
+  }
+}
+
+function globalGauge(key, value, sampleRate = 1, labels = {}) {
+  key = buildPromKey(key)
+  labels = { host: 'global', ...labels }
+  promWrapper.metric('gauge', key, labels).set(labels, sanitizeValue(value))
+}
+
+function close() {
+  for (const func of destructors) {
+    func()
+  }
+}
+
+module.exports.registerDestructor = registerDestructor
+module.exports.injectMetricsRoute = injectMetricsRoute
+module.exports.buildPromKey = buildPromKey
+module.exports.sanitizeValue = sanitizeValue
+module.exports.set = set
+module.exports.inc = inc
+module.exports.count = count
+module.exports.summary = summary
+module.exports.timing = timing
+module.exports.histogram = histogram
+module.exports.Timer = Timer
+module.exports.gauge = gauge
+module.exports.globalGauge = globalGauge
+module.exports.close = close
+module.exports.prom = promClient
+module.exports.register = promWrapper.registry
+
+module.exports.http = require('./http')
+module.exports.open_sockets = require('./open_sockets')
+module.exports.leaked_sockets = require('./leaked_sockets')
+module.exports.event_loop = require('./event_loop')
+module.exports.memory = require('./memory')
+module.exports.mongodb = require('./mongodb')

libraries/metrics/initialize.js

@@ -0,0 +1,105 @@
+/* eslint-disable no-console */
+
+/**
+ * This module initializes the metrics module. It should be imported once
+ * before any other module to support code instrumentation.
+ */
+
+const APP_NAME = process.env.METRICS_APP_NAME || 'unknown'
+const BUILD_VERSION = process.env.BUILD_VERSION
+const ENABLE_PROFILE_AGENT = process.env.ENABLE_PROFILE_AGENT === 'true'
+const GCP_OPENTELEMETRY = process.env.GCP_OPENTELEMETRY === 'true'
+const JAEGER_OPENTELEMETRY = process.env.JAEGER_OPENTELEMETRY === 'true'
+
+console.log('Initializing metrics')
+
+if (GCP_OPENTELEMETRY || JAEGER_OPENTELEMETRY) {
+  initializeOpenTelemetryInstrumentation()
+  initializeOpenTelemetryLogging()
+}
+
+if (ENABLE_PROFILE_AGENT) {
+  initializeProfileAgent()
+}
+
+initializePrometheus()
+initializePromWrapper()
+recordProcessStart()
+
+function initializeOpenTelemetryInstrumentation() {
+  console.log('Starting OpenTelemetry instrumentation')
+  const opentelemetry = require('@opentelemetry/sdk-node')
+  const {
+    getNodeAutoInstrumentations,
+  } = require('@opentelemetry/auto-instrumentations-node')
+  const { Resource } = require('@opentelemetry/resources')
+  const {
+    SemanticResourceAttributes,
+  } = require('@opentelemetry/semantic-conventions')
+
+  const resource = new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: APP_NAME,
+    [SemanticResourceAttributes.SERVICE_NAMESPACE]: 'Overleaf',
+    'host.type': 'VM',
+  })
+
+  let exporter
+  if (GCP_OPENTELEMETRY) {
+    const GCP = require('@google-cloud/opentelemetry-cloud-trace-exporter')
+    exporter = new GCP.TraceExporter()
+  } else if (JAEGER_OPENTELEMETRY) {
+    const {
+      OTLPTraceExporter,
+    } = require('@opentelemetry/exporter-trace-otlp-http')
+    exporter = new OTLPTraceExporter({
+      url: `http://${process.env.JAEGER_HOST || 'jaeger'}:4318/v1/traces`,
+    })
+  } else {
+    return
+  }
+
+  const sdk = new opentelemetry.NodeSDK({
+    traceExporter: exporter,
+    logger: console,
+    instrumentations: [getNodeAutoInstrumentations()],
+    resource,
+  })
+  sdk.start()
+}
+
+function initializeOpenTelemetryLogging() {
+  const {
+    diag,
+    DiagConsoleLogger,
+    DiagLogLevel,
+  } = require('@opentelemetry/api')
+  diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO)
+}
+
+function initializeProfileAgent() {
+  console.log('Starting Google Profile Agent')
+  const profiler = require('@google-cloud/profiler')
+  profiler.start({
+    serviceContext: {
+      service: APP_NAME,
+      version: BUILD_VERSION,
+    },
+  })
+}
+
+function initializePrometheus() {
+  const os = require('node:os')
+  const promClient = require('prom-client')
+  promClient.register.setDefaultLabels({ app: APP_NAME, host: os.hostname() })
+  promClient.collectDefaultMetrics({ timeout: 5000, prefix: '' })
+}
+
+function initializePromWrapper() {
+  const promWrapper = require('./prom_wrapper')
+  promWrapper.setupSweeping()
+}
+
+function recordProcessStart() {
+  const metrics = require('.')
+  metrics.inc('process_startup')
+}

libraries/metrics/leaked_sockets.js

@@ -0,0 +1,254 @@
+/**
+ * This file monitors HTTP connections in Node.js and logs any potential socket leaks.
+ * It uses the `diagnostics_channel` module to intercept requests and reponses in the
+ * `http` module and tracks the lifetime of each http socket. If a socket is open for
+ * longer than a specified time, it is considered a potential leak and its details are
+ * logged along with the corresponding information from /proc/net/tcp.
+ */
+
+const fs = require('node:fs')
+const diagnosticsChannel = require('node:diagnostics_channel')
+
+const SOCKET_MONITOR_INTERVAL = 60 * 1000
+// set the threshold for logging leaked sockets in minutes, defaults to 15
+const MIN_SOCKET_LEAK_TIME =
+  (parseInt(process.env.LEAKED_SOCKET_AGE_THRESHOLD, 10) || 15) * 60 * 1000
+
+// Record HTTP events using diagnostics_channel
+diagnosticsChannel.subscribe('http.client.request.start', handleRequest)
+diagnosticsChannel.subscribe('http.server.request.start', handleRequest)
+diagnosticsChannel.subscribe('http.client.response.finish', handleResponse)
+diagnosticsChannel.subscribe('http.server.response.finish', handleResponse)
+
+function handleRequest({ request: req }) {
+  const socket = req?.socket
+  if (socket) {
+    recordRequest(req, socket)
+  }
+}
+
+function recordRequest(req, socket) {
+  const { method, protocol, path, url, rawHeaders, _header } = req
+  socket._ol_debug = {
+    method,
+    protocol,
+    url: url ?? path,
+    request: { headers: rawHeaders ?? _header, ts: new Date() },
+  }
+}
+
+function handleResponse({ request: req, response: res }) {
+  const socket = req?.socket || res?.socket
+  if (!socket || !res) {
+    return
+  }
+  if (!socket._ol_debug) {
+    // I don't know if this will ever happen, but if we missed the request,
+    // record it here.
+    recordRequest(req, socket)
+  }
+  const { statusCode, statusMessage, headers, _header } = res
+  Object.assign(socket._ol_debug, {
+    response: {
+      statusCode,
+      statusMessage,
+      headers: headers ?? _header,
+      ts: new Date(),
+    },
+  })
+}
+
+// Additional functions to log request headers with sensitive information redacted
+
+function flattenHeaders(rawHeaders) {
+  // Headers can be an array [KEY, VALUE, KEY, VALUE, ..]
+  // an object {key:value, key:value, ...}
+  // or a string of the headers separated by \r\n
+  // Flatten the array and object headers into the string form.
+  if (Array.isArray(rawHeaders)) {
+    return rawHeaders
+      .map((item, index) => (index % 2 === 0 ? `${item}: ` : `${item}\r\n`))
+      .join('')
+  } else if (typeof rawHeaders === 'object') {
+    return Object.entries(rawHeaders)
+      .map(([key, value]) => `${key}: ${value}\r\n`)
+      .join('')
+  } else if (typeof rawHeaders === 'string') {
+    return rawHeaders
+  } else {
+    return JSON.stringify(rawHeaders)
+  }
+}
+
+const REDACT_REGEX = /^(Authorization|Set-Cookie|Cookie):.*?\r/gim
+
+function redactObject(obj) {
+  const result = {}
+  for (const [key, value] of Object.entries(obj)) {
+    if (value == null) {
+      result[key] = null
+    } else if (key === 'headers') {
+      // remove headers with sensitive information
+      result[key] = flattenHeaders(value).replace(
+        REDACT_REGEX,
+        `$1: REDACTED\r`
+      )
+    } else if (
+      typeof value === 'object' &&
+      ['request', 'response'].includes(key)
+    ) {
+      result[key] = redactObject(value)
+    } else {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+// Check if an old socket has crossed the threshold for logging.
+// We log multiple times with an exponential backoff so we can
+// see how long a socket hangs around.
+
+function isOldSocket(handle) {
+  const now = new Date()
+  const created = handle._ol_debug.request.ts
+  const lastLoggedAt = handle._ol_debug.lastLoggedAt ?? created
+  const nextLogTime = new Date(
+    created.getTime() +
+      Math.max(2 * (lastLoggedAt - created), MIN_SOCKET_LEAK_TIME)
+  )
+  return now >= nextLogTime
+}
+
+function logOldSocket(logger, handle, tcpinfo) {
+  const now = new Date()
+  const info = Object.assign(
+    {
+      localAddress: handle.localAddress,
+      localPort: handle.localPort,
+      remoteAddress: handle.remoteAddress,
+      remotePort: handle.remotePort,
+      tcpinfo,
+      age: Math.floor((now - handle._ol_debug.request.ts) / (60 * 1000)), // age in minutes
+    },
+    redactObject(handle._ol_debug)
+  )
+  handle._ol_debug.lastLoggedAt = now
+  if (tcpinfo) {
+    logger.error(info, 'old socket handle - tcp socket')
+  } else {
+    logger.warn(info, 'stale socket handle - no entry in /proc/net/tcp')
+  }
+}
+
+// Correlate socket handles with /proc/net/tcp entries using a key based on the
+// local and remote addresses and ports. This will allow us to distinguish between
+// sockets that are still open and sockets that have been closed and removed from
+// the /proc/net/tcp table but are still present in the node active handles array.
+
+async function getOpenSockets() {
+  // get open sockets remote and local address:port from /proc/net/tcp
+  const procNetTcp = '/proc/net/tcp'
+  const openSockets = new Map()
+  const lines = await fs.promises.readFile(procNetTcp, 'utf8')
+  for (const line of lines.split('\n')) {
+    const socket = parseProcNetTcp(line)
+    if (socket) {
+      openSockets.set(socket, line)
+    }
+  }
+  return openSockets
+}
+
+function keyFromSocket(socket) {
+  return `${socket.localAddress}:${socket.localPort} -> ${socket.remoteAddress}:${socket.remotePort}`
+}
+
+function decodeHexIpAddress(hex) {
+  // decode hex ip address to dotted decimal notation
+  const ip = parseInt(hex, 16)
+  const a = ip & 0xff
+  const b = (ip >> 8) & 0xff
+  const c = (ip >> 16) & 0xff
+  const d = (ip >> 24) & 0xff
+  return `${a}.${b}.${c}.${d}`
+}
+
+function decodeHexPort(hex) {
+  // decode hex port to decimal
+  return parseInt(hex, 16)
+}
+
+// Regex for extracting the local and remote addresses and ports from the /proc/net/tcp output
+// Example line:
+// 16: AB02A8C0:D9E2 86941864:01BB 01 00000000:00000000 02:000004BE 00000000     0        0 36802 2 0000000000000000 28 4 26 10 -1
+//     ^^^^^^^^^^^^^ ^^^^^^^^^^^^^
+//     local          remote
+
+const TCP_STATE_REGEX =
+  /^\s*\d+:\s+(?<localHexAddress>[0-9A-F]{8}):(?<localHexPort>[0-9A-F]{4})\s+(?<remoteHexAddress>[0-9A-F]{8}):(?<remoteHexPort>[0-9A-F]{4})/i
+
+function parseProcNetTcp(line) {
+  const match = line.match(TCP_STATE_REGEX)
+  if (match) {
+    const { localHexAddress, localHexPort, remoteHexAddress, remoteHexPort } =
+      match.groups
+    return keyFromSocket({
+      localAddress: decodeHexIpAddress(localHexAddress),
+      localPort: decodeHexPort(localHexPort),
+      remoteAddress: decodeHexIpAddress(remoteHexAddress),
+      remotePort: decodeHexPort(remoteHexPort),
+    })
+  }
+}
+
+let LeakedSocketsMonitor
+
+// Export the monitor and scanSockets functions
+
+module.exports = LeakedSocketsMonitor = {
+  monitor(logger) {
+    const interval = setInterval(
+      () => LeakedSocketsMonitor.scanSockets(logger),
+      SOCKET_MONITOR_INTERVAL
+    )
+    const Metrics = require('./index')
+    return Metrics.registerDestructor(() => clearInterval(interval))
+  },
+  scanSockets(logger) {
+    const debugSockets = process._getActiveHandles().filter(handle => {
+      return handle._ol_debug
+    })
+
+    // Bail out if there are no sockets with the _ol_debug property
+    if (debugSockets.length === 0) {
+      return
+    }
+
+    const oldSockets = debugSockets.filter(isOldSocket)
+
+    // Bail out if there are no old sockets to log
+    if (oldSockets.length === 0) {
+      return
+    }
+
+    // If there old sockets to log, get the connections from /proc/net/tcp
+    // to distinguish between sockets that are still open and sockets that
+    // have been closed and removed from the /proc/net/tcp table.
+    getOpenSockets()
+      .then(openSockets => {
+        oldSockets.forEach(handle => {
+          try {
+            const key = keyFromSocket(handle)
+            const tcpinfo = openSockets.get(key)
+            logOldSocket(logger, handle, tcpinfo)
+          } catch (err) {
+            logger.error({ err }, 'error in scanSockets')
+          }
+        })
+      })
+      .catch(err => {
+        logger.error({ err }, 'error getting open sockets')
+      })
+  },
+}

libraries/metrics/memory.js

@@ -0,0 +1,113 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+// record memory usage each minute and run a periodic gc(), keeping cpu
+// usage within allowable range of 1ms per minute. Also, dynamically
+// adjust the period between gc()'s to reach a target of the gc saving
+// 4 megabytes each time.
+
+let MemoryMonitor
+const oneMinute = 60 * 1000
+const oneMegaByte = 1024 * 1024
+
+let CpuTimeBucket = 100 // current cpu time allowance in milliseconds
+const CpuTimeBucketMax = 100 // maximum amount of cpu time allowed in bucket
+const CpuTimeBucketRate = 10 // add this many milliseconds per minute
+
+let gcInterval = 1 // how many minutes between gc (parameter is dynamically adjusted)
+let countSinceLastGc = 0 // how many minutes since last gc
+const MemoryChunkSize = 4 // how many megabytes we need to free to consider gc worth doing
+
+const readyToGc = function () {
+  // update allowed cpu time
+  CpuTimeBucket = CpuTimeBucket + CpuTimeBucketRate
+  CpuTimeBucket =
+    CpuTimeBucket < CpuTimeBucketMax ? CpuTimeBucket : CpuTimeBucketMax
+  // update counts since last gc
+  countSinceLastGc = countSinceLastGc + 1
+  // check there is enough time since last gc and we have enough cpu
+  return countSinceLastGc > gcInterval && CpuTimeBucket > 0
+}
+
+const executeAndTime = function (fn) {
+  // time the execution of fn() and subtract from cpu allowance
+  const t0 = process.hrtime()
+  fn()
+  const dt = process.hrtime(t0)
+  const timeTaken = (dt[0] + dt[1] * 1e-9) * 1e3 // in milliseconds
+  CpuTimeBucket -= Math.ceil(timeTaken)
+  return timeTaken
+}
+
+const inMegaBytes = function (obj) {
+  // convert process.memoryUsage hash {rss,heapTotal,heapFreed} into megabytes
+  const result = {}
+  for (const k in obj) {
+    const v = obj[k]
+    result[k] = (v / oneMegaByte).toFixed(2)
+  }
+  return result
+}
+
+const updateMemoryStats = function (oldMem, newMem) {
+  countSinceLastGc = 0
+  const delta = {}
+  for (const k in newMem) {
+    delta[k] = (newMem[k] - oldMem[k]).toFixed(2)
+  }
+  // take the max of all memory measures
+  const savedMemory = Math.max(-delta.rss, -delta.heapTotal, -delta.heapUsed)
+  delta.megabytesFreed = savedMemory
+  // did it do any good?
+  if (savedMemory < MemoryChunkSize) {
+    gcInterval = gcInterval + 1 // no, so wait longer next time
+  } else {
+    gcInterval = Math.max(gcInterval - 1, 1) // yes, wait less time
+  }
+  return delta
+}
+
+module.exports = MemoryMonitor = {
+  monitor(logger) {
+    const interval = setInterval(() => MemoryMonitor.Check(logger), oneMinute)
+    const Metrics = require('./index')
+    return Metrics.registerDestructor(() => clearInterval(interval))
+  },
+
+  Check(logger) {
+    let mem
+    const Metrics = require('./index')
+    const memBeforeGc = (mem = inMegaBytes(process.memoryUsage()))
+    Metrics.gauge('memory.rss', mem.rss)
+    Metrics.gauge('memory.heaptotal', mem.heapTotal)
+    Metrics.gauge('memory.heapused', mem.heapUsed)
+    Metrics.gauge('memory.gc-interval', gcInterval)
+    // Metrics.gauge("memory.cpu-time-bucket", CpuTimeBucket)
+
+    logger.debug(mem, 'process.memoryUsage()')
+
+    if (global.gc != null && readyToGc()) {
+      const gcTime = executeAndTime(global.gc).toFixed(2)
+      const memAfterGc = inMegaBytes(process.memoryUsage())
+      const deltaMem = updateMemoryStats(memBeforeGc, memAfterGc)
+      logger.debug(
+        {
+          gcTime,
+          memBeforeGc,
+          memAfterGc,
+          deltaMem,
+          gcInterval,
+          CpuTimeBucket,
+        },
+        'global.gc() forced'
+      )
+      // Metrics.timing("memory.gc-time", gcTime)
+      Metrics.gauge('memory.gc-rss-freed', -deltaMem.rss)
+      Metrics.gauge('memory.gc-heaptotal-freed', -deltaMem.heapTotal)
+      return Metrics.gauge('memory.gc-heapused-freed', -deltaMem.heapUsed)
+    }
+  },
+}

libraries/metrics/mongodb.js

@@ -0,0 +1,84 @@
+const { Gauge, Summary } = require('prom-client')
+
+function monitor(mongoClient) {
+  const labelNames = ['mongo_server']
+  const poolSize = new Gauge({
+    name: 'mongo_connection_pool_size',
+    help: 'number of connections in the connection pool',
+    labelNames,
+    // Use this one metric's collect() to set all metrics' values.
+    collect,
+  })
+  const availableConnections = new Gauge({
+    name: 'mongo_connection_pool_available',
+    help: 'number of connections that are not busy',
+    labelNames,
+  })
+  const waitQueueSize = new Gauge({
+    name: 'mongo_connection_pool_waiting',
+    help: 'number of operations waiting for an available connection',
+    labelNames,
+  })
+  const maxPoolSize = new Gauge({
+    name: 'mongo_connection_pool_max',
+    help: 'max size for the connection pool',
+    labelNames,
+  })
+
+  const mongoCommandTimer = new Summary({
+    name: 'mongo_command_time',
+    help: 'time taken to complete a mongo command',
+    percentiles: [],
+    labelNames: ['status', 'method'],
+  })
+
+  if (mongoClient.on) {
+    mongoClient.on('commandSucceeded', event => {
+      mongoCommandTimer.observe(
+        {
+          status: 'success',
+          method: event.commandName === 'find' ? 'read' : 'write',
+        },
+        event.duration
+      )
+    })
+
+    mongoClient.on('commandFailed', event => {
+      mongoCommandTimer.observe(
+        {
+          status: 'failed',
+          method: event.commandName === 'find' ? 'read' : 'write',
+        },
+        event.duration
+      )
+    })
+  }
+
+  function collect() {
+    // Reset all gauges in case they contain values for servers that
+    // disappeared
+    poolSize.reset()
+    availableConnections.reset()
+    waitQueueSize.reset()
+    maxPoolSize.reset()
+
+    const servers = mongoClient.topology?.s?.servers
+    if (servers != null) {
+      for (const [address, server] of servers) {
+        // The server object is different between v4 and v5 (c.f. https://github.com/mongodb/node-mongodb-native/pull/3645)
+        const pool = server.s?.pool || server.pool
+        if (pool == null) {
+          continue
+        }
+
+        const labels = { mongo_server: address }
+        poolSize.set(labels, pool.totalConnectionCount)
+        availableConnections.set(labels, pool.availableConnectionCount)
+        waitQueueSize.set(labels, pool.waitQueueSize)
+        maxPoolSize.set(labels, pool.options.maxPoolSize)
+      }
+    }
+  }
+}
+
+module.exports = { monitor }

libraries/metrics/open_sockets.js

@@ -0,0 +1,99 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let OpenSocketsMonitor
+const seconds = 1000
+
+// In Node 0.10 the default is 5, which means only 5 open connections at one.
+// Node 0.12 has a default of Infinity. Make sure we have no limit set,
+// regardless of Node version.
+require('node:http').globalAgent.maxSockets = Infinity
+require('node:https').globalAgent.maxSockets = Infinity
+
+const SOCKETS_HTTP = require('node:http').globalAgent.sockets
+const SOCKETS_HTTPS = require('node:https').globalAgent.sockets
+const FREE_SOCKETS_HTTP = require('node:http').globalAgent.freeSockets
+const FREE_SOCKETS_HTTPS = require('node:https').globalAgent.freeSockets
+
+// keep track of set gauges and reset them in the next collection cycle
+const SEEN_HOSTS_HTTP = new Set()
+const SEEN_HOSTS_HTTPS = new Set()
+const FREE_SEEN_HOSTS_HTTP = new Set()
+const FREE_SEEN_HOSTS_HTTPS = new Set()
+
+function collectConnectionsCount(
+  sockets,
+  seenHosts,
+  status,
+  https,
+  emitLegacyMetric
+) {
+  const Metrics = require('./index')
+  Object.keys(sockets).forEach(host => seenHosts.add(host))
+  seenHosts.forEach(host => {
+    // host: 'HOST:PORT:'
+    const hostname = host.split(':')[0]
+    const openConnections = (sockets[host] || []).length
+    if (!openConnections) {
+      seenHosts.delete(host)
+    }
+    Metrics.gauge('sockets', openConnections, 1, {
+      path: hostname,
+      method: https,
+      status,
+    })
+    if (status === 'open' && emitLegacyMetric) {
+      // Emit legacy metric to keep old time series intact.
+      Metrics.gauge(
+        `${status}_connections.${https}.${hostname}`,
+        openConnections
+      )
+    }
+  })
+}
+
+module.exports = OpenSocketsMonitor = {
+  monitor(emitLegacyMetric) {
+    const interval = setInterval(
+      () => OpenSocketsMonitor.gaugeOpenSockets(emitLegacyMetric),
+      5 * seconds
+    )
+    const Metrics = require('./index')
+    return Metrics.registerDestructor(() => clearInterval(interval))
+  },
+
+  gaugeOpenSockets(emitLegacyMetric) {
+    collectConnectionsCount(
+      SOCKETS_HTTP,
+      SEEN_HOSTS_HTTP,
+      'open',
+      'http',
+      emitLegacyMetric
+    )
+    collectConnectionsCount(
+      SOCKETS_HTTPS,
+      SEEN_HOSTS_HTTPS,
+      'open',
+      'https',
+      emitLegacyMetric
+    )
+    collectConnectionsCount(
+      FREE_SOCKETS_HTTP,
+      FREE_SEEN_HOSTS_HTTP,
+      'free',
+      'http',
+      false
+    )
+    collectConnectionsCount(
+      FREE_SOCKETS_HTTPS,
+      FREE_SEEN_HOSTS_HTTPS,
+      'free',
+      'https',
+      false
+    )
+  },
+}

libraries/metrics/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@overleaf/metrics",
+  "version": "4.2.0",
+  "description": "A drop-in metrics and monitoring module for node.js apps",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/overleaf/metrics-module.git"
+  },
+  "main": "index.js",
+  "dependencies": {
+    "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
+    "@google-cloud/profiler": "^6.0.0",
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.39.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.41.2",
+    "@opentelemetry/resources": "^1.15.2",
+    "@opentelemetry/sdk-node": "^0.41.2",
+    "@opentelemetry/semantic-conventions": "^1.15.2",
+    "compression": "^1.7.4",
+    "prom-client": "^14.1.1",
+    "yn": "^3.1.1"
+  },
+  "devDependencies": {
+    "bunyan": "^1.0.0",
+    "chai": "^4.3.6",
+    "mocha": "^11.1.0",
+    "sandboxed-module": "^2.0.4",
+    "sinon": "^9.2.4",
+    "typescript": "^5.0.4"
+  },
+  "scripts": {
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "test:unit": "mocha --exit test/**/*.{js,cjs}",
+    "test:acceptance": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/acceptance",
+    "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
+    "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
+    "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
+    "test:ci": "npm run test:unit",
+    "types:check": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "@overleaf/logger": "*"
+  }
+}

libraries/metrics/prom_wrapper.js

@@ -0,0 +1,177 @@
+const logger = require('@overleaf/logger')
+const prom = require('prom-client')
+const registry = require('prom-client').register
+const metrics = new Map()
+
+const labelsKey = function (labels) {
+  let keys = Object.keys(labels)
+  if (keys.length === 0) {
+    return ''
+  }
+
+  keys = keys.sort()
+
+  let hash = ''
+  for (const key of keys) {
+    if (hash.length) {
+      hash += ','
+    }
+    hash += `${key}:${labels[key]}`
+  }
+
+  return hash
+}
+
+const labelsAsArgs = function (labels, labelNames) {
+  const args = []
+  for (const label of labelNames) {
+    args.push(labels[label] || '')
+  }
+  return args
+}
+
+const PromWrapper = {
+  ttlInMinutes: 0,
+  registry,
+
+  metric(type, name, labels, buckets) {
+    return metrics.get(name) || new MetricWrapper(type, name, labels, buckets)
+  },
+
+  collectDefaultMetrics: prom.collectDefaultMetrics,
+}
+
+class MetricWrapper {
+  constructor(type, name, labels, buckets) {
+    metrics.set(name, this)
+    this.name = name
+    this.instances = new Map()
+    this.lastAccess = new Date()
+
+    const labelNames = labels ? Object.keys(labels) : []
+    switch (type) {
+      case 'counter':
+        this.metric = new prom.Counter({
+          name,
+          help: name,
+          labelNames,
+        })
+        break
+      case 'histogram':
+        this.metric = new prom.Histogram({
+          name,
+          help: name,
+          labelNames,
+          buckets,
+        })
+        break
+      case 'summary':
+        this.metric = new prom.Summary({
+          name,
+          help: name,
+          maxAgeSeconds: 60,
+          ageBuckets: 10,
+          labelNames,
+        })
+        break
+      case 'gauge':
+        this.metric = new prom.Gauge({
+          name,
+          help: name,
+          labelNames,
+        })
+        break
+      default:
+        throw new Error(`Unknown metric type: ${type}`)
+    }
+  }
+
+  inc(labels, value) {
+    this._execMethod('inc', labels, value)
+  }
+
+  observe(labels, value) {
+    this._execMethod('observe', labels, value)
+  }
+
+  set(labels, value) {
+    this._execMethod('set', labels, value)
+  }
+
+  sweep() {
+    const thresh = new Date(Date.now() - 1000 * 60 * PromWrapper.ttlInMinutes)
+    this.instances.forEach((instance, key) => {
+      if (thresh > instance.time) {
+        if (process.env.DEBUG_METRICS) {
+          // eslint-disable-next-line no-console
+          console.log(
+            'Sweeping stale metric instance',
+            this.name,
+            { labels: instance.labels },
+            key
+          )
+        }
+        this.metric.remove(
+          ...labelsAsArgs(instance.labels, this.metric.labelNames)
+        )
+      }
+    })
+
+    if (thresh > this.lastAccess) {
+      if (process.env.DEBUG_METRICS) {
+        // eslint-disable-next-line no-console
+        console.log('Sweeping stale metric', this.name, thresh, this.lastAccess)
+      }
+      metrics.delete(this.name)
+      registry.removeSingleMetric(this.name)
+    }
+  }
+
+  _execMethod(method, labels, value) {
+    const key = labelsKey(labels)
+    if (key !== '') {
+      this.instances.set(key, { time: new Date(), labels })
+    }
+    this.lastAccess = new Date()
+    try {
+      this.metric[method](labels, value)
+    } catch (err) {
+      logger.warn(
+        { err, metric: this.metric.name, labels },
+        'failed to record metric'
+      )
+    }
+  }
+}
+
+let sweepingInterval
+PromWrapper.setupSweeping = function () {
+  if (sweepingInterval) {
+    clearInterval(sweepingInterval)
+  }
+  if (!PromWrapper.ttlInMinutes) {
+    if (process.env.DEBUG_METRICS) {
+      // eslint-disable-next-line no-console
+      console.log('Not registering sweep method -- empty ttl')
+    }
+    return
+  }
+  if (process.env.DEBUG_METRICS) {
+    // eslint-disable-next-line no-console
+    console.log('Registering sweep method')
+  }
+  sweepingInterval = setInterval(function () {
+    if (process.env.DEBUG_METRICS) {
+      // eslint-disable-next-line no-console
+      console.log('Sweeping metrics')
+    }
+    metrics.forEach((metric, key) => {
+      metric.sweep()
+    })
+  }, 60000)
+
+  const Metrics = require('./index')
+  Metrics.registerDestructor(() => clearInterval(sweepingInterval))
+}
+
+module.exports = PromWrapper

libraries/metrics/test/acceptance/metrics_tests.js

@@ -0,0 +1,343 @@
+const { promisify } = require('node:util')
+const os = require('node:os')
+const http = require('node:http')
+const { expect } = require('chai')
+const Metrics = require('../..')
+
+const HOSTNAME = os.hostname()
+const APP_NAME = 'test-app'
+const sleep = promisify(setTimeout)
+
+describe('Metrics module', function () {
+  before(function () {
+    process.env.METRICS_APP_NAME = 'test-app'
+    require('../../initialize')
+  })
+
+  describe('at startup', function () {
+    it('increments the process_startup counter', async function () {
+      await expectMetricValue('process_startup', 1)
+    })
+
+    it('collects default metrics', async function () {
+      const metric = await getMetric('process_cpu_user_seconds_total')
+      expect(metric).to.exist
+    })
+  })
+
+  describe('inc()', function () {
+    it('increments counts by 1', async function () {
+      Metrics.inc('duck_count')
+      await expectMetricValue('duck_count', 1)
+      Metrics.inc('duck_count')
+      Metrics.inc('duck_count')
+      await expectMetricValue('duck_count', 3)
+    })
+
+    it('escapes special characters in the key', async function () {
+      Metrics.inc('show.me the $!!')
+      await expectMetricValue('show_me_the____', 1)
+    })
+  })
+
+  describe('count()', function () {
+    it('increments counts by the given count', async function () {
+      Metrics.count('rabbit_count', 5)
+      await expectMetricValue('rabbit_count', 5)
+      Metrics.count('rabbit_count', 6)
+      Metrics.count('rabbit_count', 7)
+      await expectMetricValue('rabbit_count', 18)
+    })
+  })
+
+  describe('summary()', function () {
+    it('collects observations', async function () {
+      Metrics.summary('oven_temp', 200)
+      Metrics.summary('oven_temp', 300)
+      Metrics.summary('oven_temp', 450)
+      const sum = await getSummarySum('oven_temp')
+      expect(sum).to.equal(950)
+    })
+  })
+
+  describe('timing()', function () {
+    it('collects timings', async function () {
+      Metrics.timing('sprint_100m', 10)
+      Metrics.timing('sprint_100m', 20)
+      Metrics.timing('sprint_100m', 30)
+      const sum = await getSummarySum('timer_sprint_100m')
+      expect(sum).to.equal(60)
+    })
+  })
+
+  describe('histogram()', function () {
+    it('collects in buckets', async function () {
+      const buckets = [10, 100, 1000]
+      Metrics.histogram('distance', 10, buckets)
+      Metrics.histogram('distance', 20, buckets)
+      Metrics.histogram('distance', 100, buckets)
+      Metrics.histogram('distance', 200, buckets)
+      Metrics.histogram('distance', 1000, buckets)
+      Metrics.histogram('distance', 2000, buckets)
+      const sum = await getSummarySum('histogram_distance')
+      expect(sum).to.equal(3330)
+      await checkHistogramValues('histogram_distance', {
+        10: 1,
+        100: 3,
+        1000: 5,
+        '+Inf': 6,
+      })
+    })
+  })
+
+  describe('Timer', function () {
+    beforeEach('collect timings', async function () {
+      const buckets = [10, 100, 1000]
+      for (const duration of [1, 1, 1, 15, 15, 15, 105, 105, 105]) {
+        const withBuckets = new Metrics.Timer(
+          'height',
+          1,
+          { label_1: 'a' },
+          buckets
+        )
+        const withOutBuckets = new Metrics.Timer('depth', 1, { label_2: 'b' })
+        await sleep(duration)
+        withBuckets.done()
+        withOutBuckets.done({ label_3: 'c' })
+      }
+    })
+
+    it('with buckets', async function () {
+      await checkHistogramValues('histogram_height', {
+        10: 3,
+        100: 6,
+        1000: 9,
+        '+Inf': 9,
+      })
+      const labelNames = await getMetric('histogram_height').labelNames
+      expect(labelNames).to.deep.equal(['label_1'])
+    })
+
+    it('without buckets', async function () {
+      await checkSummaryValues('timer_depth', {
+        0.01: 1,
+        0.05: 1,
+        0.5: 15,
+        0.9: 105,
+        0.95: 105,
+        0.99: 105,
+        0.999: 105,
+      })
+      const labelNames = await getMetric('timer_depth').labelNames
+      expect(labelNames).to.deep.equal(['label_2', 'label_3'])
+    })
+  })
+
+  describe('gauge()', function () {
+    it('records values', async function () {
+      Metrics.gauge('water_level', 1.5)
+      await expectMetricValue('water_level', 1.5)
+      Metrics.gauge('water_level', 4.2)
+      await expectMetricValue('water_level', 4.2)
+    })
+  })
+
+  describe('globalGauge()', function () {
+    it('records values without a host label', async function () {
+      Metrics.globalGauge('tire_pressure', 99.99)
+      const { value, labels } = await getMetricValue('tire_pressure')
+      expect(value).to.equal(99.99)
+      expect(labels.host).to.equal('global')
+      expect(labels.app).to.equal(APP_NAME)
+    })
+  })
+
+  describe('open_sockets', function () {
+    const keyServer1 = 'open_connections_http_127_42_42_1'
+    const keyServer2 = 'open_connections_http_127_42_42_2'
+
+    let finish1, finish2, emitResponse1, emitResponse2
+    function resetEmitResponse1() {
+      emitResponse1 = new Promise(resolve => (finish1 = resolve))
+    }
+    resetEmitResponse1()
+    function resetEmitResponse2() {
+      emitResponse2 = new Promise(resolve => (finish2 = resolve))
+    }
+    resetEmitResponse2()
+
+    let server1, server2
+    before(function setupServer1(done) {
+      server1 = http.createServer((req, res) => {
+        res.write('...')
+        emitResponse1.then(() => res.end())
+      })
+      server1.listen(0, '127.42.42.1', done)
+    })
+    before(function setupServer2(done) {
+      server2 = http.createServer((req, res) => {
+        res.write('...')
+        emitResponse2.then(() => res.end())
+      })
+      server2.listen(0, '127.42.42.2', done)
+    })
+    after(function cleanupPendingRequests() {
+      finish1()
+      finish2()
+    })
+    after(function shutdownServer1(done) {
+      if (server1) server1.close(done)
+    })
+    after(function shutdownServer2(done) {
+      if (server2) server2.close(done)
+    })
+
+    let urlServer1, urlServer2
+    before(function setUrls() {
+      urlServer1 = `http://127.42.42.1:${server1.address().port}/`
+      urlServer2 = `http://127.42.42.2:${server2.address().port}/`
+    })
+    describe('gaugeOpenSockets()', function () {
+      beforeEach(function runGaugeOpenSockets() {
+        Metrics.open_sockets.gaugeOpenSockets(true)
+      })
+
+      describe('without pending connections', function () {
+        it('emits no open_connections', async function () {
+          await expectNoMetricValue(keyServer1)
+          await expectNoMetricValue(keyServer2)
+        })
+      })
+
+      describe('with pending connections for server1', function () {
+        before(function (done) {
+          http.get(urlServer1)
+          http.get(urlServer1)
+          setTimeout(done, 10)
+        })
+
+        it('emits 2 open_connections for server1', async function () {
+          await expectMetricValue(keyServer1, 2)
+        })
+
+        it('emits no open_connections for server2', async function () {
+          await expectNoMetricValue(keyServer2)
+        })
+      })
+
+      describe('with pending connections for server1 and server2', function () {
+        before(function (done) {
+          http.get(urlServer2)
+          http.get(urlServer2)
+          setTimeout(done, 10)
+        })
+
+        it('emits 2 open_connections for server1', async function () {
+          await expectMetricValue(keyServer1, 2)
+        })
+
+        it('emits 2 open_connections for server2', async function () {
+          await expectMetricValue(keyServer2, 2)
+        })
+      })
+
+      describe('when requests finish for server1', function () {
+        before(function (done) {
+          finish1()
+          resetEmitResponse1()
+          http.get(urlServer1)
+
+          setTimeout(done, 10)
+        })
+
+        it('emits 1 open_connections for server1', async function () {
+          await expectMetricValue(keyServer1, 1)
+        })
+
+        it('emits 2 open_connections for server2', async function () {
+          await expectMetricValue(keyServer2, 2)
+        })
+      })
+
+      describe('when all requests complete', function () {
+        before(function (done) {
+          finish1()
+          finish2()
+
+          setTimeout(done, 10)
+        })
+
+        it('emits no open_connections', async function () {
+          await expectNoMetricValue(keyServer1)
+          await expectNoMetricValue(keyServer2)
+        })
+      })
+    })
+  })
+})
+
+function getMetric(key) {
+  return Metrics.register.getSingleMetric(key)
+}
+
+async function getSummarySum(key) {
+  const metric = getMetric(key)
+  const item = await metric.get()
+  for (const value of item.values) {
+    if (value.metricName === `${key}_sum`) {
+      return value.value
+    }
+  }
+  return null
+}
+
+async function checkHistogramValues(key, values) {
+  const metric = getMetric(key)
+  const item = await metric.get()
+  const found = {}
+  for (const value of item.values) {
+    const bucket = value.labels.le
+    if (!bucket) continue
+    found[bucket] = value.value
+  }
+  expect(found).to.deep.equal(values)
+  return null
+}
+
+async function checkSummaryValues(key, values) {
+  const metric = getMetric(key)
+  const item = await metric.get()
+  const found = {}
+  for (const value of item.values) {
+    const quantile = value.labels.quantile
+    if (!quantile) continue
+    found[quantile] = value.value
+  }
+  for (const quantile of Object.keys(values)) {
+    expect(found[quantile]).to.be.within(
+      values[quantile] - 5,
+      values[quantile] + 15,
+      `quantile: ${quantile}`
+    )
+  }
+  return null
+}
+
+async function getMetricValue(key) {
+  const metrics = await Metrics.register.getMetricsAsJSON()
+  const metric = metrics.find(m => m.name === key)
+  return metric.values[0]
+}
+
+async function expectMetricValue(key, expectedValue) {
+  const value = await getMetricValue(key)
+  expect(value.value).to.equal(expectedValue)
+  expect(value.labels.host).to.equal(HOSTNAME)
+  expect(value.labels.app).to.equal(APP_NAME)
+}
+
+async function expectNoMetricValue(key) {
+  const metric = getMetric(key)
+  if (!metric) return
+  await expectMetricValue(key, 0)
+}

libraries/metrics/test/unit/js/event_loop.js

@@ -0,0 +1,44 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const { expect } = chai
+const path = require('node:path')
+const modulePath = path.join(__dirname, '../../../event_loop.js')
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+
+describe('event_loop', function () {
+  before(function () {
+    this.metrics = {
+      timing: sinon.stub(),
+      registerDestructor: sinon.stub(),
+    }
+    this.logger = {
+      warn: sinon.stub(),
+    }
+    return (this.event_loop = SandboxedModule.require(modulePath, {
+      requires: {
+        './index': this.metrics,
+      },
+    }))
+  })
+
+  describe('with a logger provided', function () {
+    before(function () {
+      return this.event_loop.monitor(this.logger)
+    })
+
+    return it('should register a destructor with metrics', function () {
+      return expect(this.metrics.registerDestructor.called).to.equal(true)
+    })
+  })
+
+  return describe('without a logger provided', function () {
+    return it('should throw an exception', function () {
+      return expect(this.event_loop.monitor).to.throw('logger is undefined')
+    })
+  })
+})

libraries/metrics/test/unit/js/http.js

@@ -0,0 +1,171 @@
+const Path = require('node:path')
+const SandboxedModule = require('sandboxed-module')
+const sinon = require('sinon')
+
+const MODULE_PATH = Path.join(__dirname, '../../../http.js')
+
+describe('http.monitor', function () {
+  beforeEach(function () {
+    this.req = {
+      method: 'POST',
+      url: '/project/1234/cleanup',
+      headers: {
+        'content-length': '123',
+      },
+      route: {
+        path: '/project/:id/cleanup',
+      },
+    }
+    this.originalResponseEnd = sinon.stub()
+    this.res = {
+      end: this.originalResponseEnd,
+    }
+    this.data = 'data'
+    this.logger = {
+      debug: sinon.stub(),
+      info: sinon.stub(),
+      warn: sinon.stub(),
+    }
+    this.Metrics = {
+      timing: sinon.stub(),
+      summary: sinon.stub(),
+    }
+    this.clock = sinon.useFakeTimers()
+
+    this.http = SandboxedModule.require(MODULE_PATH, {
+      requires: {
+        './index': this.Metrics,
+      },
+    })
+  })
+
+  afterEach(function () {
+    this.clock.restore()
+  })
+
+  describe('with the default options', function () {
+    beforeEach('set up the monitor', function (done) {
+      this.http.monitor(this.logger)(this.req, this.res, done)
+    })
+
+    describe('after a simple request', function () {
+      endRequest()
+      expectOriginalEndCalled()
+      expectMetrics()
+
+      it('logs the request at the DEBUG level', function () {
+        sinon.assert.calledWith(
+          this.logger.debug,
+          { req: this.req, res: this.res, responseTimeMs: 500 },
+          '%s %s',
+          this.req.method,
+          this.req.url
+        )
+      })
+    })
+
+    describe('when logging is disabled', function () {
+      beforeEach('disable logging', function () {
+        this.req.logger.disable()
+      })
+
+      endRequest()
+      expectOriginalEndCalled()
+      expectMetrics()
+
+      it("doesn't log the request", function () {
+        sinon.assert.notCalled(this.logger.debug)
+      })
+    })
+
+    describe('with custom log fields', function () {
+      beforeEach('add custom fields', function () {
+        this.req.logger.addFields({ a: 1, b: 2 })
+      })
+
+      endRequest()
+
+      it('logs the request with the custom log fields', function () {
+        sinon.assert.calledWith(
+          this.logger.debug,
+          { req: this.req, res: this.res, responseTimeMs: 500, a: 1, b: 2 },
+          '%s %s',
+          this.req.method,
+          this.req.url
+        )
+      })
+    })
+
+    describe('when setting the log level', function () {
+      beforeEach('set custom level', function () {
+        this.req.logger.setLevel('warn')
+      })
+
+      endRequest()
+
+      it('logs the request at the custom level', function () {
+        sinon.assert.calledWith(
+          this.logger.warn,
+          { req: this.req, res: this.res, responseTimeMs: 500 },
+          '%s %s',
+          this.req.method,
+          this.req.url
+        )
+      })
+    })
+  })
+
+  describe('with a different default log level', function () {
+    beforeEach('set up the monitor', function (done) {
+      this.http.monitor(this.logger, 'info')(this.req, this.res, done)
+    })
+
+    endRequest()
+
+    it('logs the request at that level', function () {
+      sinon.assert.calledWith(
+        this.logger.info,
+        { req: this.req, res: this.res, responseTimeMs: 500 },
+        '%s %s',
+        this.req.method,
+        this.req.url
+      )
+    })
+  })
+})
+
+function endRequest() {
+  beforeEach('end the request', function () {
+    this.clock.tick(500)
+    this.res.end(this.data)
+  })
+}
+
+function expectOriginalEndCalled() {
+  it('calls the original res.end()', function () {
+    sinon.assert.calledWith(this.originalResponseEnd, this.data)
+  })
+}
+
+function expectMetrics() {
+  it('records the response time', function () {
+    sinon.assert.calledWith(this.Metrics.timing, 'http_request', 500, null, {
+      method: this.req.method,
+      status_code: this.res.status_code,
+      path: 'project_id_cleanup',
+    })
+  })
+
+  it('records the request size', function () {
+    sinon.assert.calledWith(
+      this.Metrics.summary,
+      'http_request_size_bytes',
+      123,
+      {
+        method: this.req.method,
+        status_code: this.res.status_code,
+        path: 'project_id_cleanup',
+      }
+    )
+  })
+}

libraries/metrics/test/unit/js/mongodb.js

@@ -0,0 +1,89 @@
+const Metrics = require('../../..')
+
+const { expect } = require('chai')
+const prom = require('prom-client')
+
+describe('mongodb', function () {
+  beforeEach(function () {
+    prom.register.clear()
+    this.pool = {
+      totalConnectionCount: 8,
+      availableConnectionCount: 2,
+      waitQueueSize: 4,
+      options: { maxPoolSize: 10 },
+    }
+    this.servers = new Map([['server1', { s: { pool: this.pool } }]])
+
+    this.mongoClient = { topology: { s: { servers: this.servers } } }
+  })
+
+  it('handles an unconnected client', async function () {
+    const mongoClient = {}
+    Metrics.mongodb.monitor(mongoClient)
+    const metrics = await getMetrics()
+    expect(metrics).to.deep.equal({})
+  })
+
+  it('collects Mongo metrics', async function () {
+    Metrics.mongodb.monitor(this.mongoClient)
+    const metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+    })
+  })
+
+  it('handles topology changes', async function () {
+    Metrics.mongodb.monitor(this.mongoClient)
+    let metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+    })
+
+    // Add a server
+    this.servers.set('server2', this.servers.get('server1'))
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server1': 10,
+      'mongo_connection_pool_size:server1': 8,
+      'mongo_connection_pool_available:server1': 2,
+      'mongo_connection_pool_waiting:server1': 4,
+      'mongo_connection_pool_max:server2': 10,
+      'mongo_connection_pool_size:server2': 8,
+      'mongo_connection_pool_available:server2': 2,
+      'mongo_connection_pool_waiting:server2': 4,
+    })
+
+    // Delete a server
+    this.servers.delete('server1')
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({
+      'mongo_connection_pool_max:server2': 10,
+      'mongo_connection_pool_size:server2': 8,
+      'mongo_connection_pool_available:server2': 2,
+      'mongo_connection_pool_waiting:server2': 4,
+    })
+
+    // Delete another server
+    this.servers.delete('server2')
+    metrics = await getMetrics()
+    expect(metrics).to.deep.equal({})
+  })
+})
+
+async function getMetrics() {
+  const metrics = await prom.register.getMetricsAsJSON()
+  const result = {}
+  for (const metric of metrics) {
+    for (const value of metric.values) {
+      const key = `${metric.name}:${value.labels.mongo_server}`
+      result[key] = value.value
+    }
+  }
+  return result
+}

libraries/metrics/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.backend.json",
+  "include": [
+    "**/*.js",
+    "**/*.cjs"
+  ]
+}

libraries/metrics/uv_threadpool_size.js

@@ -0,0 +1,5 @@
+if (!process.env.UV_THREADPOOL_SIZE) {
+  process.env.UV_THREADPOOL_SIZE = 16
+  // eslint-disable-next-line no-console
+  console.log(`Set UV_THREADPOOL_SIZE=${process.env.UV_THREADPOOL_SIZE}`)
+}