first commit

services/chat/app/js/Features/Messages/MessageFormatter.js

@@ -0,0 +1,60 @@
+export function formatMessageForClientSide(message) {
+  if (message._id) {
+    message.id = message._id.toString()
+    delete message._id
+  }
+  const formattedMessage = {
+    id: message.id,
+    content: message.content,
+    timestamp: message.timestamp,
+    user_id: message.user_id,
+  }
+  if (message.edited_at) {
+    formattedMessage.edited_at = message.edited_at
+  }
+  return formattedMessage
+}
+
+export function formatMessagesForClientSide(messages) {
+  return messages.map(message => formatMessageForClientSide(message))
+}
+
+export function groupMessagesByThreads(rooms, messages) {
+  let room, thread
+  const roomsById = {}
+  for (room of rooms) {
+    roomsById[room._id.toString()] = room
+  }
+
+  const threads = {}
+  const getThread = function (room) {
+    const threadId = room.thread_id.toString()
+    if (threads[threadId]) {
+      return threads[threadId]
+    } else {
+      const thread = { messages: [] }
+      if (room.resolved) {
+        thread.resolved = true
+        thread.resolved_at = room.resolved.ts
+        thread.resolved_by_user_id = room.resolved.user_id
+      }
+      threads[threadId] = thread
+      return thread
+    }
+  }
+
+  for (const message of messages) {
+    room = roomsById[message.room_id.toString()]
+    if (room) {
+      thread = getThread(room)
+      thread.messages.push(formatMessageForClientSide(message))
+    }
+  }
+
+  for (const threadId in threads) {
+    thread = threads[threadId]
+    thread.messages.sort((a, b) => a.timestamp - b.timestamp)
+  }
+
+  return threads
+}

services/chat/app/js/Features/Messages/MessageHttpController.js

@@ -0,0 +1,313 @@
+import logger from '@overleaf/logger'
+import * as MessageManager from './MessageManager.js'
+import * as MessageFormatter from './MessageFormatter.js'
+import * as ThreadManager from '../Threads/ThreadManager.js'
+import { ObjectId } from '../../mongodb.js'
+
+const DEFAULT_MESSAGE_LIMIT = 50
+const MAX_MESSAGE_LENGTH = 10 * 1024 // 10kb, about 1,500 words
+
+function readContext(context, req) {
+  req.body = context.requestBody
+  req.params = context.params.path
+  req.query = context.params.query
+  if (typeof req.params.projectId !== 'undefined') {
+    if (!ObjectId.isValid(req.params.projectId)) {
+      context.res.status(400).setBody('Invalid projectId')
+    }
+  }
+  if (typeof req.params.threadId !== 'undefined') {
+    if (!ObjectId.isValid(req.params.threadId)) {
+      context.res.status(400).setBody('Invalid threadId')
+    }
+  }
+}
+
+/**
+ * @param context
+ * @param {(req: unknown, res: unknown) => Promise<unknown>} ControllerMethod
+ * @returns {Promise<*>}
+ */
+export async function callMessageHttpController(context, ControllerMethod) {
+  const req = {}
+  readContext(context, req)
+  if (context.res.statusCode !== 400) {
+    return await ControllerMethod(req, context.res)
+  } else {
+    return context.res.body
+  }
+}
+
+export async function getGlobalMessages(context) {
+  return await callMessageHttpController(context, _getGlobalMessages)
+}
+
+export async function sendGlobalMessage(context) {
+  return await callMessageHttpController(context, _sendGlobalMessage)
+}
+
+export async function sendMessage(context) {
+  return await callMessageHttpController(context, _sendThreadMessage)
+}
+
+export async function getThreads(context) {
+  return await callMessageHttpController(context, _getAllThreads)
+}
+
+export async function resolveThread(context) {
+  return await callMessageHttpController(context, _resolveThread)
+}
+
+export async function reopenThread(context) {
+  return await callMessageHttpController(context, _reopenThread)
+}
+
+export async function deleteThread(context) {
+  return await callMessageHttpController(context, _deleteThread)
+}
+
+export async function editMessage(context) {
+  return await callMessageHttpController(context, _editMessage)
+}
+
+export async function deleteMessage(context) {
+  return await callMessageHttpController(context, _deleteMessage)
+}
+
+export async function deleteUserMessage(context) {
+  return await callMessageHttpController(context, _deleteUserMessage)
+}
+
+export async function getResolvedThreadIds(context) {
+  return await callMessageHttpController(context, _getResolvedThreadIds)
+}
+
+export async function destroyProject(context) {
+  return await callMessageHttpController(context, _destroyProject)
+}
+
+export async function duplicateCommentThreads(context) {
+  return await callMessageHttpController(context, _duplicateCommentThreads)
+}
+
+export async function generateThreadData(context) {
+  return await callMessageHttpController(context, _generateThreadData)
+}
+
+export async function getStatus(context) {
+  const message = 'chat is alive'
+  context.res.status(200).setBody(message)
+  return message
+}
+
+const _getGlobalMessages = async (req, res) => {
+  await _getMessages(ThreadManager.GLOBAL_THREAD, req, res)
+}
+
+async function _sendGlobalMessage(req, res) {
+  const { user_id: userId, content } = req.body
+  const { projectId } = req.params
+  return await _sendMessage(
+    userId,
+    projectId,
+    content,
+    ThreadManager.GLOBAL_THREAD,
+    res
+  )
+}
+
+async function _sendThreadMessage(req, res) {
+  const { user_id: userId, content } = req.body
+  const { projectId, threadId } = req.params
+  return await _sendMessage(userId, projectId, content, threadId, res)
+}
+
+const _getAllThreads = async (req, res) => {
+  const { projectId } = req.params
+  logger.debug({ projectId }, 'getting all threads')
+  const rooms = await ThreadManager.findAllThreadRooms(projectId)
+  const roomIds = rooms.map(r => r._id)
+  const messages = await MessageManager.findAllMessagesInRooms(roomIds)
+  const threads = MessageFormatter.groupMessagesByThreads(rooms, messages)
+  res.json(threads)
+}
+
+const _generateThreadData = async (req, res) => {
+  const { projectId } = req.params
+  const { threads } = req.body
+  logger.debug({ projectId }, 'getting all threads')
+  const rooms = await ThreadManager.findThreadsById(projectId, threads)
+  const roomIds = rooms.map(r => r._id)
+  const messages = await MessageManager.findAllMessagesInRooms(roomIds)
+  logger.debug({ rooms, messages }, 'looked up messages in the rooms')
+  const threadData = MessageFormatter.groupMessagesByThreads(rooms, messages)
+  res.json(threadData)
+}
+
+const _resolveThread = async (req, res) => {
+  const { projectId, threadId } = req.params
+  const { user_id: userId } = req.body
+  logger.debug({ userId, projectId, threadId }, 'marking thread as resolved')
+  await ThreadManager.resolveThread(projectId, threadId, userId)
+  res.status(204)
+}
+
+const _reopenThread = async (req, res) => {
+  const { projectId, threadId } = req.params
+  logger.debug({ projectId, threadId }, 'reopening thread')
+  await ThreadManager.reopenThread(projectId, threadId)
+  res.status(204)
+}
+
+const _deleteThread = async (req, res) => {
+  const { projectId, threadId } = req.params
+  logger.debug({ projectId, threadId }, 'deleting thread')
+  const roomId = await ThreadManager.deleteThread(projectId, threadId)
+  await MessageManager.deleteAllMessagesInRoom(roomId)
+  res.status(204)
+}
+
+const _editMessage = async (req, res) => {
+  const { content, userId } = req.body
+  const { projectId, threadId, messageId } = req.params
+  logger.debug({ projectId, threadId, messageId, content }, 'editing message')
+  const room = await ThreadManager.findOrCreateThread(projectId, threadId)
+  const found = await MessageManager.updateMessage(
+    room._id,
+    messageId,
+    userId,
+    content,
+    Date.now()
+  )
+  if (!found) {
+    res.status(404)
+    return
+  }
+  res.status(204)
+}
+
+const _deleteMessage = async (req, res) => {
+  const { projectId, threadId, messageId } = req.params
+  logger.debug({ projectId, threadId, messageId }, 'deleting message')
+  const room = await ThreadManager.findOrCreateThread(projectId, threadId)
+  await MessageManager.deleteMessage(room._id, messageId)
+  res.status(204)
+}
+
+const _deleteUserMessage = async (req, res) => {
+  const { projectId, threadId, userId, messageId } = req.params
+  const room = await ThreadManager.findOrCreateThread(projectId, threadId)
+  await MessageManager.deleteUserMessage(userId, room._id, messageId)
+  res.status(204)
+}
+
+const _getResolvedThreadIds = async (req, res) => {
+  const { projectId } = req.params
+  const resolvedThreadIds = await ThreadManager.getResolvedThreadIds(projectId)
+  res.json({ resolvedThreadIds })
+}
+
+const _destroyProject = async (req, res) => {
+  const { projectId } = req.params
+  logger.debug({ projectId }, 'destroying project')
+  const rooms = await ThreadManager.findAllThreadRoomsAndGlobalThread(projectId)
+  const roomIds = rooms.map(r => r._id)
+  logger.debug({ projectId, roomIds }, 'deleting all messages in rooms')
+  await MessageManager.deleteAllMessagesInRooms(roomIds)
+  logger.debug({ projectId }, 'deleting all threads in project')
+  await ThreadManager.deleteAllThreadsInProject(projectId)
+  res.status(204)
+}
+
+async function _sendMessage(userId, projectId, content, clientThreadId, res) {
+  if (!ObjectId.isValid(userId)) {
+    const message = 'Invalid userId'
+    res.status(400).setBody(message)
+    return message
+  }
+  if (!content) {
+    const message = 'No content provided'
+    res.status(400).setBody(message)
+    return message
+  }
+  if (content.length > MAX_MESSAGE_LENGTH) {
+    const message = `Content too long (> ${MAX_MESSAGE_LENGTH} bytes)`
+    res.status(400).setBody(message)
+    return message
+  }
+  logger.debug(
+    { clientThreadId, projectId, userId, content },
+    'new message received'
+  )
+  const thread = await ThreadManager.findOrCreateThread(
+    projectId,
+    clientThreadId
+  )
+  let message = await MessageManager.createMessage(
+    thread._id,
+    userId,
+    content,
+    Date.now()
+  )
+  message = MessageFormatter.formatMessageForClientSide(message)
+  message.room_id = projectId
+  res.status(201).setBody(message)
+}
+
+async function _getMessages(clientThreadId, req, res) {
+  let before, limit
+  const { projectId } = req.params
+  if (req.query.before) {
+    before = parseInt(req.query.before, 10)
+  } else {
+    before = null
+  }
+  if (req.query.limit) {
+    limit = parseInt(req.query.limit, 10)
+  } else {
+    limit = DEFAULT_MESSAGE_LIMIT
+  }
+  logger.debug(
+    { limit, before, projectId, clientThreadId },
+    'get message request received'
+  )
+  const thread = await ThreadManager.findOrCreateThread(
+    projectId,
+    clientThreadId
+  )
+  const threadObjectId = thread._id
+  logger.debug(
+    { limit, before, projectId, clientThreadId, threadObjectId },
+    'found or created thread'
+  )
+  let messages = await MessageManager.getMessages(threadObjectId, limit, before)
+  messages = MessageFormatter.formatMessagesForClientSide(messages)
+  logger.debug({ projectId, messages }, 'got messages')
+  res.status(200).setBody(messages)
+}
+
+async function _duplicateCommentThreads(req, res) {
+  const { projectId } = req.params
+  const { threads } = req.body
+  const result = {}
+  for (const id of threads) {
+    logger.debug({ projectId, thread: id }, 'duplicating thread')
+    try {
+      const { oldRoom, newRoom } = await ThreadManager.duplicateThread(
+        projectId,
+        id
+      )
+      await MessageManager.duplicateRoomToOtherRoom(oldRoom._id, newRoom._id)
+      result[id] = { duplicateId: newRoom.thread_id }
+    } catch (error) {
+      if (error instanceof ThreadManager.MissingThreadError) {
+        // Expected error when the comment has been deleted prior to duplication
+        result[id] = { error: 'not found' }
+      } else {
+        logger.err({ error }, 'error duplicating thread')
+        result[id] = { error: 'unknown' }
+      }
+    }
+  }
+  res.json({ newThreads: result })
+}

services/chat/app/js/Features/Messages/MessageManager.js

@@ -0,0 +1,112 @@
+import { db, ObjectId } from '../../mongodb.js'
+
+export async function createMessage(roomId, userId, content, timestamp) {
+  let newMessageOpts = {
+    content,
+    room_id: roomId,
+    user_id: userId,
+    timestamp,
+  }
+  newMessageOpts = _ensureIdsAreObjectIds(newMessageOpts)
+  const confirmation = await db.messages.insertOne(newMessageOpts)
+  newMessageOpts._id = confirmation.insertedId
+  return newMessageOpts
+}
+
+export async function getMessages(roomId, limit, before) {
+  let query = { room_id: roomId }
+  if (before) {
+    query.timestamp = { $lt: before }
+  }
+  query = _ensureIdsAreObjectIds(query)
+  return await db.messages
+    .find(query)
+    .sort({ timestamp: -1 })
+    .limit(limit)
+    .toArray()
+}
+
+export async function findAllMessagesInRooms(roomIds) {
+  return await db.messages
+    .find({
+      room_id: { $in: roomIds },
+    })
+    .toArray()
+}
+
+export async function deleteAllMessagesInRoom(roomId) {
+  await db.messages.deleteMany({
+    room_id: roomId,
+  })
+}
+
+export async function deleteAllMessagesInRooms(roomIds) {
+  await db.messages.deleteMany({
+    room_id: { $in: roomIds },
+  })
+}
+
+export async function updateMessage(
+  roomId,
+  messageId,
+  userId,
+  content,
+  timestamp
+) {
+  const query = _ensureIdsAreObjectIds({
+    _id: messageId,
+    room_id: roomId,
+  })
+  if (userId) {
+    query.user_id = new ObjectId(userId)
+  }
+  const res = await db.messages.updateOne(query, {
+    $set: {
+      content,
+      edited_at: timestamp,
+    },
+  })
+  return res.modifiedCount === 1
+}
+
+export async function deleteMessage(roomId, messageId) {
+  const query = _ensureIdsAreObjectIds({
+    _id: messageId,
+    room_id: roomId,
+  })
+  await db.messages.deleteOne(query)
+}
+
+export async function deleteUserMessage(userId, roomId, messageId) {
+  await db.messages.deleteOne({
+    _id: new ObjectId(messageId),
+    user_id: new ObjectId(userId),
+    room_id: new ObjectId(roomId),
+  })
+}
+
+function _ensureIdsAreObjectIds(query) {
+  if (query.user_id && !(query.user_id instanceof ObjectId)) {
+    query.user_id = new ObjectId(query.user_id)
+  }
+  if (query.room_id && !(query.room_id instanceof ObjectId)) {
+    query.room_id = new ObjectId(query.room_id)
+  }
+  if (query._id && !(query._id instanceof ObjectId)) {
+    query._id = new ObjectId(query._id)
+  }
+  return query
+}
+
+export async function duplicateRoomToOtherRoom(sourceRoomId, targetRoomId) {
+  const sourceMessages = await findAllMessagesInRooms([sourceRoomId])
+  const targetMessages = sourceMessages.map(comment => {
+    return _ensureIdsAreObjectIds({
+      room_id: targetRoomId,
+      content: comment.content,
+      timestamp: comment.timestamp,
+      user_id: comment.user_id,
+    })
+  })
+  await db.messages.insertMany(targetMessages)
+}

services/chat/app/js/Features/Threads/ThreadManager.js

@@ -0,0 +1,157 @@
+import { db, ObjectId } from '../../mongodb.js'
+
+export class MissingThreadError extends Error {}
+
+export const GLOBAL_THREAD = 'GLOBAL'
+
+export async function findOrCreateThread(projectId, threadId) {
+  let query, update
+  projectId = new ObjectId(projectId.toString())
+  if (threadId !== GLOBAL_THREAD) {
+    threadId = new ObjectId(threadId.toString())
+  }
+
+  if (threadId === GLOBAL_THREAD) {
+    query = {
+      project_id: projectId,
+      thread_id: { $exists: false },
+    }
+    update = {
+      project_id: projectId,
+    }
+  } else {
+    query = {
+      project_id: projectId,
+      thread_id: threadId,
+    }
+    update = {
+      project_id: projectId,
+      thread_id: threadId,
+    }
+  }
+
+  const result = await db.rooms.findOneAndUpdate(
+    query,
+    { $set: update },
+    { upsert: true, returnDocument: 'after' }
+  )
+  return result
+}
+
+export async function findAllThreadRooms(projectId) {
+  return await db.rooms
+    .find(
+      {
+        project_id: new ObjectId(projectId.toString()),
+        thread_id: { $exists: true },
+      },
+      {
+        thread_id: 1,
+        resolved: 1,
+      }
+    )
+    .toArray()
+}
+
+export async function findAllThreadRoomsAndGlobalThread(projectId) {
+  return await db.rooms
+    .find(
+      {
+        project_id: new ObjectId(projectId.toString()),
+      },
+      {
+        thread_id: 1,
+        resolved: 1,
+      }
+    )
+    .toArray()
+}
+
+export async function resolveThread(projectId, threadId, userId) {
+  await db.rooms.updateOne(
+    {
+      project_id: new ObjectId(projectId.toString()),
+      thread_id: new ObjectId(threadId.toString()),
+    },
+    {
+      $set: {
+        resolved: {
+          user_id: userId,
+          ts: new Date(),
+        },
+      },
+    }
+  )
+}
+
+export async function reopenThread(projectId, threadId) {
+  await db.rooms.updateOne(
+    {
+      project_id: new ObjectId(projectId.toString()),
+      thread_id: new ObjectId(threadId.toString()),
+    },
+    {
+      $unset: {
+        resolved: true,
+      },
+    }
+  )
+}
+
+export async function deleteThread(projectId, threadId) {
+  const room = await findOrCreateThread(projectId, threadId)
+  await db.rooms.deleteOne({
+    _id: room._id,
+  })
+  return room._id
+}
+
+export async function deleteAllThreadsInProject(projectId) {
+  await db.rooms.deleteMany({
+    project_id: new ObjectId(projectId.toString()),
+  })
+}
+
+export async function getResolvedThreadIds(projectId) {
+  const resolvedThreadIds = await db.rooms
+    .find(
+      {
+        project_id: new ObjectId(projectId),
+        thread_id: { $exists: true },
+        resolved: { $exists: true },
+      },
+      { projection: { thread_id: 1 } }
+    )
+    .map(record => record.thread_id.toString())
+    .toArray()
+  return resolvedThreadIds
+}
+
+export async function duplicateThread(projectId, threadId) {
+  const room = await db.rooms.findOne({
+    project_id: new ObjectId(projectId),
+    thread_id: new ObjectId(threadId),
+  })
+  if (!room) {
+    throw new MissingThreadError('Trying to duplicate a non-existent thread')
+  }
+  const newRoom = {
+    project_id: room.project_id,
+    thread_id: new ObjectId(),
+  }
+  if (room.resolved) {
+    newRoom.resolved = room.resolved
+  }
+  const confirmation = await db.rooms.insertOne(newRoom)
+  newRoom._id = confirmation.insertedId
+  return { oldRoom: room, newRoom }
+}
+
+export async function findThreadsById(projectId, threadIds) {
+  return await db.rooms
+    .find({
+      project_id: new ObjectId(projectId),
+      thread_id: { $in: threadIds.map(id => new ObjectId(id)) },
+    })
+    .toArray()
+}

services/chat/app/js/mongodb.js

@@ -0,0 +1,18 @@
+import Metrics from '@overleaf/metrics'
+import Settings from '@overleaf/settings'
+import { MongoClient } from 'mongodb'
+
+export { ObjectId } from 'mongodb'
+
+export const mongoClient = new MongoClient(
+  Settings.mongo.url,
+  Settings.mongo.options
+)
+const mongoDb = mongoClient.db()
+
+export const db = {
+  messages: mongoDb.collection('messages'),
+  rooms: mongoDb.collection('rooms'),
+}
+
+Metrics.mongodb.monitor(mongoClient)

services/chat/app/js/server.js

@@ -0,0 +1,51 @@
+import http from 'node:http'
+import metrics from '@overleaf/metrics'
+import logger from '@overleaf/logger'
+import express from 'express'
+import exegesisExpress from 'exegesis-express'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import * as messagesController from './Features/Messages/MessageHttpController.js'
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+
+logger.initialize('chat')
+metrics.open_sockets.monitor()
+
+metrics.leaked_sockets.monitor(logger)
+
+export async function createServer() {
+  const app = express()
+
+  app.use(metrics.http.monitor(logger))
+  metrics.injectMetricsRoute(app)
+
+  // See https://github.com/exegesis-js/exegesis/blob/master/docs/Options.md
+  const options = {
+    controllers: { messagesController },
+    ignoreServers: true,
+    allowMissingControllers: false,
+  }
+
+  // const exegesisMiddleware = await exegesisExpress.middleware(
+  const exegesisMiddleware = await exegesisExpress.middleware(
+    path.resolve(__dirname, '../../chat.yaml'),
+    options
+  )
+
+  // If you have any body parsers, this should go before them.
+  app.use(exegesisMiddleware)
+
+  // Return a 404
+  app.use((req, res) => {
+    res.status(404).json({ message: `Not found` })
+  })
+
+  // Handle any unexpected errors
+  app.use((err, req, res, next) => {
+    res.status(500).json({ message: `Internal error: ${err.message}` })
+  })
+
+  const server = http.createServer(app)
+  return { app, server }
+}

services/chat/app/js/util/promises.js

@@ -0,0 +1,10 @@
+/**
+ * Transform an async function into an Express middleware
+ *
+ * Any error will be passed to the error middlewares via `next()`
+ */
+export function expressify(fn) {
+  return (req, res, next) => {
+    fn(req, res, next).catch(next)
+  }
+}