first commit

libraries/redis-wrapper/.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+max_line_length = 79
+tab_width = 4
+trim_trailing_whitespace = true

libraries/redis-wrapper/.gitignore

@@ -0,0 +1,13 @@
+**.swp
+
+app.js
+app/js/
+test/unit/js/
+public/build/
+
+node_modules/
+
+/public/js/chat.js
+plato/
+
+.npmrc

libraries/redis-wrapper/.mocharc.json

@@ -0,0 +1,3 @@
+{
+  "require": "test/setup.js"
+}

libraries/redis-wrapper/.nvmrc

@@ -0,0 +1 @@
+20.18.2

libraries/redis-wrapper/Errors.js

@@ -0,0 +1,15 @@
+const OError = require('@overleaf/o-error')
+
+class RedisError extends OError {}
+class RedisHealthCheckFailed extends RedisError {}
+class RedisHealthCheckTimedOut extends RedisHealthCheckFailed {}
+class RedisHealthCheckWriteError extends RedisHealthCheckFailed {}
+class RedisHealthCheckVerifyError extends RedisHealthCheckFailed {}
+
+module.exports = {
+  RedisError,
+  RedisHealthCheckFailed,
+  RedisHealthCheckTimedOut,
+  RedisHealthCheckWriteError,
+  RedisHealthCheckVerifyError,
+}

libraries/redis-wrapper/RedisLocker.js

@@ -0,0 +1,225 @@
+const { promisify } = require('node:util')
+const metrics = require('@overleaf/metrics')
+const logger = require('@overleaf/logger')
+const os = require('node:os')
+const crypto = require('node:crypto')
+
+const HOST = os.hostname()
+const PID = process.pid
+const RND = crypto.randomBytes(4).toString('hex')
+let COUNT = 0
+
+const MAX_REDIS_REQUEST_LENGTH = 5000 // 5 seconds
+
+const UNLOCK_SCRIPT =
+  'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end'
+
+module.exports = class RedisLocker {
+  /**
+   * @param {import('ioredis')} rclient initialized ioredis client
+   * @param {function(string): string} getKey compose the redis key based on the passed id
+   * @param {function(Error, string): Error} wrapTimeoutError assign the id to a designated field on the error
+   * @param {string} metricsPrefix prefix all the metrics with the given prefix
+   * @param {number} lockTTLSeconds
+   *
+   * @example ```
+   * const lock = new RedisLocker({
+   *   rclient,
+   *   getKey(userId) { return `blocking:{userId}` },
+   *   wrapTimeoutError(err, userId) { err.userId = userId; return err },
+   *   metricsPrefix: 'user',
+   * })
+   *
+   * lock.getLock(user._id, (err, value) => {
+   *   if (err) return callback(err)
+   *   // do work
+   *   lock.releaseLock(user._id, callback)
+   * }
+   * ```
+   */
+  constructor({
+    rclient,
+    getKey,
+    wrapTimeoutError,
+    metricsPrefix,
+    lockTTLSeconds = 30,
+  }) {
+    if (
+      typeof lockTTLSeconds !== 'number' ||
+      lockTTLSeconds < 30 ||
+      lockTTLSeconds >= 1000
+    ) {
+      // set upper limit to 1000s to detect wrong units
+      throw new Error('redis lock TTL must be at least 30s and below 1000s')
+    }
+
+    this.rclient = rclient
+    this.getKey = getKey
+    this.wrapTimeoutError = wrapTimeoutError
+    this.metricsPrefix = metricsPrefix
+
+    this.LOCK_TEST_INTERVAL = 50 // 50ms between each test of the lock
+    this.MAX_TEST_INTERVAL = 1000 // back off to 1s between each test of the lock
+    this.MAX_LOCK_WAIT_TIME = 10000 // 10s maximum time to spend trying to get the lock
+    this.LOCK_TTL = lockTTLSeconds // seconds. Time until lock auto expires in redis.
+
+    // read-only copy for unit tests
+    this.unlockScript = UNLOCK_SCRIPT
+
+    this.promises = {
+      checkLock: promisify(this.checkLock.bind(this)),
+      getLock: promisify(this.getLock.bind(this)),
+      releaseLock: promisify(this.releaseLock.bind(this)),
+
+      // tryLock returns two values: gotLock and lockValue. We need to merge
+      // these two values into one for the promises version.
+      tryLock: id =>
+        new Promise((resolve, reject) => {
+          this.tryLock(id, (err, gotLock, lockValue) => {
+            if (err) {
+              reject(err)
+            } else if (!gotLock) {
+              resolve(null)
+            } else {
+              resolve(lockValue)
+            }
+          })
+        }),
+    }
+  }
+
+  // Use a signed lock value as described in
+  // https://redis.io/docs/reference/patterns/distributed-locks/#correct-implementation-with-a-single-instance
+  // to prevent accidental unlocking by multiple processes
+  randomLock() {
+    const time = Date.now()
+    return `locked:host=${HOST}:pid=${PID}:random=${RND}:time=${time}:count=${COUNT++}`
+  }
+
+  /**
+   * @param {Callback} callback
+   */
+  tryLock(id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    const lockValue = this.randomLock()
+    const key = this.getKey(id)
+    const startTime = Date.now()
+    return this.rclient.set(
+      key,
+      lockValue,
+      'EX',
+      this.LOCK_TTL,
+      'NX',
+      (err, gotLock) => {
+        if (err != null) {
+          return callback(err)
+        }
+        if (gotLock === 'OK') {
+          metrics.inc(this.metricsPrefix + '-not-blocking')
+          const timeTaken = Date.now() - startTime
+          if (timeTaken > MAX_REDIS_REQUEST_LENGTH) {
+            // took too long, so try to free the lock
+            return this.releaseLock(id, lockValue, function (err, result) {
+              if (err != null) {
+                return callback(err)
+              } // error freeing lock
+              return callback(null, false)
+            }) // tell caller they didn't get the lock
+          } else {
+            return callback(null, true, lockValue)
+          }
+        } else {
+          metrics.inc(this.metricsPrefix + '-blocking')
+          return callback(null, false)
+        }
+      }
+    )
+  }
+
+  /**
+   * @param {Callback} callback
+   */
+  getLock(id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    const startTime = Date.now()
+    let testInterval = this.LOCK_TEST_INTERVAL
+    const attempt = () => {
+      if (Date.now() - startTime > this.MAX_LOCK_WAIT_TIME) {
+        const e = this.wrapTimeoutError(new Error('Timeout'), id)
+        return callback(e)
+      }
+
+      return this.tryLock(id, (error, gotLock, lockValue) => {
+        if (error != null) {
+          return callback(error)
+        }
+        if (gotLock) {
+          return callback(null, lockValue)
+        } else {
+          setTimeout(attempt, testInterval)
+          // back off when the lock is taken to avoid overloading
+          return (testInterval = Math.min(
+            testInterval * 2,
+            this.MAX_TEST_INTERVAL
+          ))
+        }
+      })
+    }
+    attempt()
+  }
+
+  /**
+   * @param {Callback} callback
+   */
+  checkLock(id, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    const key = this.getKey(id)
+    return this.rclient.exists(key, (err, exists) => {
+      if (err != null) {
+        return callback(err)
+      }
+      exists = parseInt(exists)
+      if (exists === 1) {
+        metrics.inc(this.metricsPrefix + '-blocking')
+        return callback(null, false)
+      } else {
+        metrics.inc(this.metricsPrefix + '-not-blocking')
+        return callback(null, true)
+      }
+    })
+  }
+
+  /**
+   * @param {Callback} callback
+   */
+  releaseLock(id, lockValue, callback) {
+    const key = this.getKey(id)
+    return this.rclient.eval(
+      UNLOCK_SCRIPT,
+      1,
+      key,
+      lockValue,
+      (err, result) => {
+        if (err != null) {
+          return callback(err)
+        } else if (result != null && result !== 1) {
+          // successful unlock should release exactly one key
+          logger.error(
+            { id, key, lockValue, redis_err: err, redis_result: result },
+            'unlocking error'
+          )
+          metrics.inc(this.metricsPrefix + '-unlock-error')
+          return callback(new Error('tried to release timed out lock'))
+        } else {
+          return callback(null, result)
+        }
+      }
+    )
+  }
+}

libraries/redis-wrapper/RedisWebLocker.js

@@ -0,0 +1,207 @@
+const { callbackify, promisify } = require('node:util')
+const metrics = require('@overleaf/metrics')
+const logger = require('@overleaf/logger')
+const os = require('node:os')
+const crypto = require('node:crypto')
+const async = require('async')
+
+const HOST = os.hostname()
+const PID = process.pid
+const RND = crypto.randomBytes(4).toString('hex')
+let COUNT = 0
+
+const LOCK_QUEUES = new Map() // queue lock requests for each name/id so they get the lock on a first-come first-served basis
+
+const UNLOCK_SCRIPT =
+  'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end'
+
+module.exports = class RedisWebLocker {
+  constructor({ rclient, getKey, options }) {
+    this.rclient = rclient
+    this.getKey = getKey
+
+    // ms between each test of the lock
+    this.LOCK_TEST_INTERVAL = options.lockTestInterval || 50
+    // back off to ms between each test of the lock
+    this.MAX_TEST_INTERVAL = options.maxTestInterval || 1000
+    // ms maximum time to spend trying to get the lock
+    this.MAX_LOCK_WAIT_TIME = options.maxLockWaitTime || 10000
+    // seconds. Time until lock auto expires in redis
+    this.REDIS_LOCK_EXPIRY = options.redisLockExpiry || 30
+    // ms, if execution takes longer than this then log
+    this.SLOW_EXECUTION_THRESHOLD = options.slowExecutionThreshold || 5000
+    // read-only copy for unit tests
+    this.unlockScript = UNLOCK_SCRIPT
+
+    const promisifiedRunWithLock = promisify(this.runWithLock).bind(this)
+    this.promises = {
+      runWithLock(namespace, id, runner) {
+        const cbRunner = callbackify(runner)
+        return promisifiedRunWithLock(namespace, id, cbRunner)
+      },
+    }
+  }
+
+  // Use a signed lock value as described in
+  // http://redis.io/topics/distlock#correct-implementation-with-a-single-instance
+  // to prevent accidental unlocking by multiple processes
+  randomLock() {
+    const time = Date.now()
+    return `locked:host=${HOST}:pid=${PID}:random=${RND}:time=${time}:count=${COUNT++}`
+  }
+
+  runWithLock(namespace, id, runner, callback) {
+    // runner must be a function accepting a callback, e.g. runner = (cb) ->
+
+    // This error is defined here so we get a useful stacktrace
+    const slowExecutionError = new Error('slow execution during lock')
+
+    const timer = new metrics.Timer(`lock.${namespace}`)
+    const key = this.getKey(namespace, id)
+    this._getLock(key, namespace, (error, lockValue) => {
+      if (error != null) {
+        return callback(error)
+      }
+
+      // The lock can expire in redis but the process carry on. This setTimeout call
+      // is designed to log if this happens.
+      function countIfExceededLockTimeout() {
+        metrics.inc(`lock.${namespace}.exceeded_lock_timeout`)
+        logger.debug('exceeded lock timeout', {
+          namespace,
+          id,
+          slowExecutionError,
+        })
+      }
+      const exceededLockTimeout = setTimeout(
+        countIfExceededLockTimeout,
+        this.REDIS_LOCK_EXPIRY * 1000
+      )
+
+      runner((error1, ...values) =>
+        this._releaseLock(key, lockValue, error2 => {
+          clearTimeout(exceededLockTimeout)
+
+          const timeTaken = new Date() - timer.start
+          if (timeTaken > this.SLOW_EXECUTION_THRESHOLD) {
+            logger.debug('slow execution during lock', {
+              namespace,
+              id,
+              timeTaken,
+              slowExecutionError,
+            })
+          }
+
+          timer.done()
+          error = error1 || error2
+          if (error != null) {
+            return callback(error)
+          }
+          callback(null, ...values)
+        })
+      )
+    })
+  }
+
+  _tryLock(key, namespace, callback) {
+    const lockValue = this.randomLock()
+    this.rclient.set(
+      key,
+      lockValue,
+      'EX',
+      this.REDIS_LOCK_EXPIRY,
+      'NX',
+      (err, gotLock) => {
+        if (err != null) {
+          return callback(err)
+        }
+        if (gotLock === 'OK') {
+          metrics.inc(`lock.${namespace}.try.success`)
+          callback(err, true, lockValue)
+        } else {
+          metrics.inc(`lock.${namespace}.try.failed`)
+          logger.debug({ key, redis_response: gotLock }, 'lock is locked')
+          callback(err, false)
+        }
+      }
+    )
+  }
+
+  // it's sufficient to serialize within a process because that is where the parallel operations occur
+  _getLock(key, namespace, callback) {
+    // this is what we need to do for each lock we want to request
+    const task = next =>
+      this._getLockByPolling(key, namespace, (error, lockValue) => {
+        // tell the queue to start trying to get the next lock (if any)
+        next()
+        // we have got a lock result, so we can continue with our own execution
+        callback(error, lockValue)
+      })
+    // create a queue for this key if needed
+    const queueName = `${key}:${namespace}`
+    let queue = LOCK_QUEUES.get(queueName)
+    if (queue == null) {
+      const handler = (fn, cb) => fn(cb)
+      // set up a new queue for this key
+      queue = async.queue(handler, 1)
+      queue.push(task)
+      // remove the queue object when queue is empty
+      queue.drain(() => {
+        LOCK_QUEUES.delete(queueName)
+      })
+      // store the queue in our global map
+      LOCK_QUEUES.set(queueName, queue)
+    } else {
+      // queue the request to get the lock
+      queue.push(task)
+    }
+  }
+
+  _getLockByPolling(key, namespace, callback) {
+    const startTime = Date.now()
+    const testInterval = this.LOCK_TEST_INTERVAL
+    let attempts = 0
+    const attempt = () => {
+      if (Date.now() - startTime > this.MAX_LOCK_WAIT_TIME) {
+        metrics.inc(`lock.${namespace}.get.failed`)
+        return callback(new Error('Timeout'))
+      }
+
+      attempts += 1
+      this._tryLock(key, namespace, (error, gotLock, lockValue) => {
+        if (error != null) {
+          return callback(error)
+        }
+        if (gotLock) {
+          metrics.gauge(`lock.${namespace}.get.success.tries`, attempts)
+          callback(null, lockValue)
+        } else {
+          setTimeout(attempt, testInterval)
+        }
+      })
+    }
+    attempt()
+  }
+
+  _releaseLock(key, lockValue, callback) {
+    this.rclient.eval(this.unlockScript, 1, key, lockValue, (err, result) => {
+      if (err != null) {
+        callback(err)
+      } else if (result != null && result !== 1) {
+        // successful unlock should release exactly one key
+        logger.warn(
+          { key, lockValue, redis_err: err, redis_result: result },
+          'unlocking error'
+        )
+        metrics.inc('unlock-error')
+        callback(new Error('tried to release timed out lock'))
+      } else {
+        callback(null, result)
+      }
+    })
+  }
+
+  _lockQueuesSize() {
+    return LOCK_QUEUES.size
+  }
+}

libraries/redis-wrapper/buildscript.txt

@@ -0,0 +1,10 @@
+redis-wrapper
+--dependencies=None
+--docker-repos=gcr.io/overleaf-ops
+--env-add=
+--env-pass-through=
+--esmock-loader=False
+--is-library=True
+--node-version=20.18.2
+--public-repo=False
+--script-version=4.7.0

libraries/redis-wrapper/index.js

@@ -0,0 +1,173 @@
+const crypto = require('node:crypto')
+const os = require('node:os')
+const { promisify } = require('node:util')
+
+const Redis = require('ioredis')
+
+const {
+  RedisHealthCheckTimedOut,
+  RedisHealthCheckWriteError,
+  RedisHealthCheckVerifyError,
+} = require('./Errors')
+
+const HEARTBEAT_TIMEOUT = 2000
+
+// generate unique values for health check
+const HOST = os.hostname()
+const PID = process.pid
+const RND = crypto.randomBytes(4).toString('hex')
+let COUNT = 0
+
+function createClient(opts) {
+  const standardOpts = Object.assign({}, opts)
+  delete standardOpts.key_schema
+
+  if (standardOpts.retry_max_delay == null) {
+    standardOpts.retry_max_delay = 5000 // ms
+  }
+
+  if (standardOpts.endpoints) {
+    throw new Error(
+      '@overleaf/redis-wrapper: redis-sentinel is no longer supported'
+    )
+  }
+
+  let client
+  if (standardOpts.cluster) {
+    delete standardOpts.cluster
+    client = new Redis.Cluster(opts.cluster, standardOpts)
+  } else {
+    client = new Redis(standardOpts)
+  }
+  monkeyPatchIoRedisExec(client)
+  client.healthCheck = callback => {
+    if (callback) {
+      // callback based invocation
+      healthCheck(client).then(callback).catch(callback)
+    } else {
+      // Promise based invocation
+      return healthCheck(client)
+    }
+  }
+  return client
+}
+
+async function healthCheck(client) {
+  // check the redis connection by storing and retrieving a unique key/value pair
+  const uniqueToken = `host=${HOST}:pid=${PID}:random=${RND}:time=${Date.now()}:count=${COUNT++}`
+
+  // o-error context
+  const context = {
+    uniqueToken,
+    stage: 'add context for a timeout',
+  }
+
+  await runWithTimeout({
+    runner: runCheck(client, uniqueToken, context),
+    timeout: HEARTBEAT_TIMEOUT,
+    context,
+  })
+}
+
+async function runCheck(client, uniqueToken, context) {
+  const healthCheckKey = `_redis-wrapper:healthCheckKey:{${uniqueToken}}`
+  const healthCheckValue = `_redis-wrapper:healthCheckValue:{${uniqueToken}}`
+
+  // set the unique key/value pair
+  context.stage = 'write'
+  const writeAck = await client
+    .set(healthCheckKey, healthCheckValue, 'EX', 60)
+    .catch(err => {
+      throw new RedisHealthCheckWriteError('write errored', context, err)
+    })
+  if (writeAck !== 'OK') {
+    context.writeAck = writeAck
+    throw new RedisHealthCheckWriteError('write failed', context)
+  }
+
+  // check that we can retrieve the unique key/value pair
+  context.stage = 'verify'
+  const [roundTrippedHealthCheckValue, deleteAck] = await client
+    .multi()
+    .get(healthCheckKey)
+    .del(healthCheckKey)
+    .exec()
+    .catch(err => {
+      throw new RedisHealthCheckVerifyError(
+        'read/delete errored',
+        context,
+        err
+      )
+    })
+  if (roundTrippedHealthCheckValue !== healthCheckValue) {
+    context.roundTrippedHealthCheckValue = roundTrippedHealthCheckValue
+    throw new RedisHealthCheckVerifyError('read failed', context)
+  }
+  if (deleteAck !== 1) {
+    context.deleteAck = deleteAck
+    throw new RedisHealthCheckVerifyError('delete failed', context)
+  }
+}
+
+function unwrapMultiResult(result, callback) {
+  // ioredis exec returns a results like:
+  // [ [null, 42], [null, "foo"] ]
+  // where the first entries in each 2-tuple are
+  // presumably errors for each individual command,
+  // and the second entry is the result. We need to transform
+  // this into the same result as the old redis driver:
+  // [ 42, "foo" ]
+  //
+  // Basically reverse:
+  // https://github.com/luin/ioredis/blob/v4.17.3/lib/utils/index.ts#L75-L92
+  const filteredResult = []
+  for (const [err, value] of result || []) {
+    if (err) {
+      return callback(err)
+    } else {
+      filteredResult.push(value)
+    }
+  }
+  callback(null, filteredResult)
+}
+const unwrapMultiResultPromisified = promisify(unwrapMultiResult)
+
+function monkeyPatchIoRedisExec(client) {
+  const _multi = client.multi
+  client.multi = function () {
+    const multi = _multi.apply(client, arguments)
+    const _exec = multi.exec
+    multi.exec = callback => {
+      if (callback) {
+        // callback based invocation
+        _exec.call(multi, (error, result) => {
+          // The command can fail all-together due to syntax errors
+          if (error) return callback(error)
+          unwrapMultiResult(result, callback)
+        })
+      } else {
+        // Promise based invocation
+        return _exec.call(multi).then(unwrapMultiResultPromisified)
+      }
+    }
+    return multi
+  }
+}
+
+async function runWithTimeout({ runner, timeout, context }) {
+  let healthCheckDeadline
+  await Promise.race([
+    new Promise((resolve, reject) => {
+      healthCheckDeadline = setTimeout(() => {
+        // attach the timeout when hitting the timeout only
+        context.timeout = timeout
+        reject(new RedisHealthCheckTimedOut('timeout', context))
+      }, timeout)
+    }),
+    runner.finally(() => clearTimeout(healthCheckDeadline)),
+  ])
+}
+
+module.exports = {
+  createClient,
+}

libraries/redis-wrapper/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@overleaf/redis-wrapper",
+  "version": "2.1.0",
+  "description": "Redis wrapper for node which will either use cluster or single instance redis",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "Errors.js",
+    "RedisLocker.js",
+    "RedisWebLocker.js"
+  ],
+  "author": "Overleaf (https://www.overleaf.com)",
+  "repository": "github:overleaf/redis-wrapper",
+  "license": "ISC",
+  "scripts": {
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
+    "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
+    "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
+    "test:ci": "npm run test:unit",
+    "test:unit": "mocha --exit test/**/*.{js,cjs}",
+    "types:check": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "@overleaf/logger": "*",
+    "@overleaf/metrics": "*",
+    "@overleaf/o-error": "*"
+  },
+  "dependencies": {
+    "async": "^3.2.5",
+    "ioredis": "~4.27.1"
+  },
+  "devDependencies": {
+    "@overleaf/logger": "*",
+    "@overleaf/o-error": "*",
+    "chai": "^4.3.6",
+    "mocha": "^11.1.0",
+    "sandboxed-module": "^2.0.4",
+    "sinon": "^9.2.4",
+    "typescript": "^5.0.4"
+  }
+}

libraries/redis-wrapper/test/scripts/cluster/clear-dbs.sh

@@ -0,0 +1,4 @@
+while true; do
+  seq 0 8 \
+  | xargs -I% redis-cli -p 700% FLUSHALL > /dev/null
+done

libraries/redis-wrapper/test/scripts/cluster/cluster.js

@@ -0,0 +1,26 @@
+/*
+  execute this script with a redis cluster running to test the health check.
+  starting and stopping shards with this script running is a good test.
+
+  to create a new cluster, use $ ./create-redis-cluster.sh
+  to run a chaos monkey, use $ ./clear-dbs.sh
+*/
+
+const redis = require('../../../')
+const logger = require('@overleaf/logger')
+
+const rclient = redis.createClient({
+  cluster: Array.from({ length: 9 }).map((value, index) => {
+    return { host: '127.0.0.1', port: 7000 + index }
+  }),
+})
+
+setInterval(() => {
+  rclient.healthCheck(err => {
+    if (err) {
+      logger.error({ err }, 'HEALTH CHECK FAILED')
+    } else {
+      logger.info('HEALTH CHECK OK')
+    }
+  })
+}, 1000)

libraries/redis-wrapper/test/scripts/cluster/create-cluster.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# USAGE: $0 [NUMBER_OF_NODES, default: 9] [DATA_DIR, default: a new temp dir]
+#
+# ports are assigned from 7000 on
+#
+# NOTE: the cluster setup requires redis 5+
+
+set -ex
+
+COUNT=${1:-9}
+DATA=$2
+
+if [[ -z "$DATA" ]]; then
+  IS_TEMP=1
+  TEMP=`mktemp -d`
+  DATA="$TEMP"
+fi
+
+HAS_DATA=
+if [[ -e "$DATA/7000/node.conf" ]]; then
+    HAS_DATA=1
+fi
+
+PIDs=""
+
+cleanup() {
+  # ensure that we delete the temp dir, no matter how the kill cmd exists
+  set +e
+  # invoke kill with at least one PID
+  echo "$PIDs" | xargs -r kill
+  if [[ ! -z "$IS_TEMP" ]]; then
+    rm -rf "$TEMP"
+  fi
+}
+trap cleanup exit
+
+for NUM in `seq "$COUNT"`; do
+  PORT=`expr 6999 + "$NUM"`
+  CWD="$DATA/$PORT"
+  mkdir -p "$CWD"
+  pushd "$CWD"
+  redis-server \
+    --appendonly no \
+    --cluster-enabled yes \
+    --cluster-config-file node.conf \
+    --port "$PORT" \
+    --save "" \
+    > /dev/null \
+  &
+  PIDs="$PIDs $!"
+  popd
+done
+
+# initial nodes
+if [[ -z "$HAS_DATA" ]]; then
+  # confirm the setup
+  echo yes \
+  | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002
+fi
+
+# scale up as requested
+for NUM in `seq 4 "$COUNT"`; do
+  PORT=`expr 6999 + "$NUM"`
+  GUARD="$DATA/$PORT/.joined"
+  if [[ ! -e "$GUARD" ]]; then
+    redis-cli --cluster add-node "127.0.0.1:$PORT" 127.0.0.1:7000 --cluster-slave
+    touch "$GUARD"
+  fi
+done
+
+echo "CLUSTER IS READY" >&2
+wait

libraries/redis-wrapper/test/scripts/standalone.js

@@ -0,0 +1,17 @@
+// execute this script with a redis container running to test the health check
+// starting and stopping redis with this script running is a good test
+
+const redis = require('../../')
+const logger = require('@overleaf/logger')
+
+const rclient = redis.createClient({ host: '127.0.0.1', port: '6379' })
+
+setInterval(() => {
+  rclient.healthCheck(err => {
+    if (err) {
+      logger.error({ err }, 'HEALTH CHECK FAILED')
+    } else {
+      logger.info('HEALTH CHECK OK')
+    }
+  })
+}, 1000)

libraries/redis-wrapper/test/setup.js

@@ -0,0 +1,9 @@
+const SandboxedModule = require('sandboxed-module')
+
+SandboxedModule.configure({
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
+})

libraries/redis-wrapper/test/unit/src/test.js

@@ -0,0 +1,219 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS206: Consider reworking classes to avoid initClass
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+require('chai').should()
+const SandboxedModule = require('sandboxed-module')
+const assert = require('node:assert')
+const path = require('node:path')
+const sinon = require('sinon')
+const modulePath = path.join(__dirname, './../../../index.js')
+const redisLockerModulePath = path.join(__dirname, './../../../RedisLocker.js')
+const { expect } = require('chai')
+
+describe('index', function () {
+  beforeEach(function () {
+    let Cluster, IoRedis, ioredisConstructor
+    this.settings = {}
+    this.ioredisConstructor = ioredisConstructor = sinon.stub()
+
+    this.ioredis = IoRedis = (function () {
+      let createIoRedis
+      IoRedis = class IoRedis {
+        static initClass() {
+          this.prototype.on = sinon.stub()
+          createIoRedis = ioredisConstructor
+        }
+
+        constructor() {
+          return createIoRedis.apply(this, arguments)
+        }
+      }
+      IoRedis.initClass()
+      return IoRedis
+    })()
+    this.ioredis.Cluster = Cluster = (function () {
+      Cluster = class Cluster {
+        static initClass() {
+          this.prototype.on = sinon.stub()
+        }
+
+        constructor(config, options) {
+          this.config = config
+          this.options = options
+        }
+      }
+      Cluster.initClass()
+      return Cluster
+    })()
+    this.redis = SandboxedModule.require(modulePath, {
+      requires: {
+        ioredis: this.ioredis,
+      },
+      globals: {
+        process,
+        Buffer,
+      },
+    })
+    this.auth_pass = '1234 pass'
+
+    this.RedisLocker = SandboxedModule.require(redisLockerModulePath, {
+      requires: {
+        '@overleaf/metrics': {
+          inc() {},
+        },
+      },
+      globals: {
+        process,
+        Math,
+        Buffer,
+      },
+    })
+  })
+
+  describe('lock TTL', function () {
+    it('should throw an error when creating a client with wrong type', function () {
+      const createNewRedisLock = () => {
+        return new this.RedisLocker({
+          lockTTLSeconds: '60',
+        })
+      }
+      expect(createNewRedisLock).to.throw(
+        'redis lock TTL must be at least 30s and below 1000s'
+      )
+    })
+
+    it('should throw an error when creating a client with small TTL', function () {
+      const createNewRedisLock = () => {
+        return new this.RedisLocker({
+          lockTTLSeconds: 1,
+        })
+      }
+      expect(createNewRedisLock).to.throw(
+        'redis lock TTL must be at least 30s and below 1000s'
+      )
+    })
+
+    it('should throw an error when creating a client with huge TTL', function () {
+      const createNewRedisLock = () => {
+        return new this.RedisLocker({
+          lockTTLSeconds: 30_000,
+        })
+      }
+      expect(createNewRedisLock).to.throw(
+        'redis lock TTL must be at least 30s and below 1000s'
+      )
+    })
+  })
+
+  describe('redis-sentinel', function () {
+    it('should throw an error when creating a client', function () {
+      const redisSentinelOptions = {
+        endpoints: ['127.0.0.1:1234', '127.0.0.1:2345', '127.0.0.1:3456'],
+      }
+      const createNewClient = () => {
+        this.redis.createClient(redisSentinelOptions)
+      }
+      expect(createNewClient).to.throw(
+        '@overleaf/redis-wrapper: redis-sentinel is no longer supported'
+      )
+    })
+  })
+
+  describe('single node redis', function () {
+    beforeEach(function () {
+      return (this.standardOpts = {
+        auth_pass: this.auth_pass,
+        port: 1234,
+        host: 'redis.mysite.env',
+      })
+    })
+
+    it('should work without opts', function () {
+      this.redis.createClient()
+    })
+
+    it('should use the ioredis driver in single-instance mode if a non array is passed', function () {
+      const client = this.redis.createClient(this.standardOpts)
+      return assert.equal(client.constructor, this.ioredis)
+    })
+
+    return it('should call createClient for the ioredis driver in single-instance mode if a non array is passed', function () {
+      this.redis.createClient(this.standardOpts)
+      return this.ioredisConstructor
+        .calledWith(sinon.match(this.standardOpts))
+        .should.equal(true)
+    })
+  })
+
+  describe('cluster', function () {
+    beforeEach(function () {
+      this.cluster = [{ mock: 'cluster' }, { mock: 'cluster2' }]
+      this.extraOptions = { keepAlive: 100 }
+      return (this.settings = {
+        cluster: this.cluster,
+        redisOptions: this.extraOptions,
+        key_schema: {
+          foo(x) {
+            return `${x}`
+          },
+        },
+      })
+    })
+
+    it('should pass the options correctly though with no options', function () {
+      const client = this.redis.createClient({ cluster: this.cluster })
+      assert(client instanceof this.ioredis.Cluster)
+      return client.config.should.deep.equal(this.cluster)
+    })
+
+    it('should not pass the key_schema through to the driver', function () {
+      const client = this.redis.createClient({
+        cluster: this.cluster,
+        key_schema: 'foobar',
+      })
+      assert(client instanceof this.ioredis.Cluster)
+      client.config.should.deep.equal(this.cluster)
+      return expect(client.options).to.deep.equal({ retry_max_delay: 5000 })
+    })
+
+    return it('should pass the options correctly though with additional options', function () {
+      const client = this.redis.createClient(this.settings)
+      assert(client instanceof this.ioredis.Cluster)
+      client.config.should.deep.equal(this.cluster)
+      // need to use expect here because of _.clone in sandbox
+      return expect(client.options).to.deep.equal({
+        redisOptions: this.extraOptions,
+        retry_max_delay: 5000,
+      })
+    })
+  })
+
+  return describe('monkey patch ioredis exec', function () {
+    beforeEach(function () {
+      this.callback = sinon.stub()
+      this.results = []
+      this.multiOrig = { exec: sinon.stub().yields(null, this.results) }
+      this.client = { multi: sinon.stub().returns(this.multiOrig) }
+      this.ioredisConstructor.returns(this.client)
+      this.redis.createClient(this.client)
+      return (this.multi = this.client.multi())
+    })
+
+    it('should return the old redis format for an array', function () {
+      this.results[0] = [null, 42]
+      this.results[1] = [null, 'foo']
+      this.multi.exec(this.callback)
+      return this.callback.calledWith(null, [42, 'foo']).should.equal(true)
+    })
+
+    return it('should return the old redis format when there is an error', function () {
+      this.results[0] = [null, 42]
+      this.results[1] = ['error', 'foo']
+      this.multi.exec(this.callback)
+      return this.callback.calledWith('error').should.equal(true)
+    })
+  })
+})

libraries/redis-wrapper/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.backend.json",
+  "include": [
+    "**/*.js",
+    "**/*.cjs"
+  ]
+}