first commit

2025-04-24 13:11:28 +08:00
commit ff9c54d5e4
5960 changed files with 834111 additions and 0 deletions
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
@@ -0,0 +1,42 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const app = require('../../../../app')
+
+module.exports = {
+  running: false,
+  initing: false,
+  callbacks: [],
+  ensureRunning(callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    if (this.running) {
+      return callback()
+    } else if (this.initing) {
+      return this.callbacks.push(callback)
+    }
+    this.initing = true
+    this.callbacks.push(callback)
+    app.listen(3003, '127.0.0.1', error => {
+      if (error != null) {
+        throw error
+      }
+      this.running = true
+      return (() => {
+        const result = []
+        for (callback of Array.from(this.callbacks)) {
+          result.push(callback())
+        }
+        return result
+      })()
+    })
+  },
+}
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
@@ -0,0 +1,246 @@
+let DocUpdaterClient
+const Settings = require('@overleaf/settings')
+const _ = require('lodash')
+const rclient = require('@overleaf/redis-wrapper').createClient(
+  Settings.redis.documentupdater
+)
+const keys = Settings.redis.documentupdater.key_schema
+const request = require('request').defaults({ jar: false })
+const async = require('async')
+
+const rclientSub = require('@overleaf/redis-wrapper').createClient(
+  Settings.redis.pubsub
+)
+rclientSub.subscribe('applied-ops')
+rclientSub.setMaxListeners(0)
+
+module.exports = DocUpdaterClient = {
+  randomId() {
+    let str = ''
+    for (let i = 0; i < 24; i++) {
+      str += Math.floor(Math.random() * 16).toString(16)
+    }
+    return str
+  },
+
+  subscribeToAppliedOps(callback) {
+    rclientSub.on('message', callback)
+  },
+
+  _getPendingUpdateListKey() {
+    const shard = _.random(0, Settings.dispatcherCount - 1)
+    if (shard === 0) {
+      return 'pending-updates-list'
+    } else {
+      return `pending-updates-list-${shard}`
+    }
+  },
+
+  sendUpdate(projectId, docId, update, callback) {
+    rclient.rpush(
+      keys.pendingUpdates({ doc_id: docId }),
+      JSON.stringify(update),
+      error => {
+        if (error) {
+          return callback(error)
+        }
+        const docKey = `${projectId}:${docId}`
+        rclient.sadd('DocsWithPendingUpdates', docKey, error => {
+          if (error) {
+            return callback(error)
+          }
+
+          rclient.rpush(
+            DocUpdaterClient._getPendingUpdateListKey(),
+            docKey,
+            callback
+          )
+        })
+      }
+    )
+  },
+
+  sendUpdates(projectId, docId, updates, callback) {
+    DocUpdaterClient.preloadDoc(projectId, docId, error => {
+      if (error) {
+        return callback(error)
+      }
+      const jobs = updates.map(update => callback => {
+        DocUpdaterClient.sendUpdate(projectId, docId, update, callback)
+      })
+      async.series(jobs, err => {
+        if (err) {
+          return callback(err)
+        }
+        DocUpdaterClient.waitForPendingUpdates(projectId, docId, callback)
+      })
+    })
+  },
+
+  waitForPendingUpdates(projectId, docId, callback) {
+    async.retry(
+      { times: 30, interval: 100 },
+      cb =>
+        rclient.llen(keys.pendingUpdates({ doc_id: docId }), (err, length) => {
+          if (err) {
+            return cb(err)
+          }
+          if (length > 0) {
+            cb(new Error('updates still pending'))
+          } else {
+            cb()
+          }
+        }),
+      callback
+    )
+  },
+
+  getDoc(projectId, docId, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
+  getDocAndRecentOps(projectId, docId, fromVersion, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
+  getProjectLastUpdatedAt(projectId, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
+  preloadDoc(projectId, docId, callback) {
+    DocUpdaterClient.getDoc(projectId, docId, callback)
+  },
+
+  peekDoc(projectId, docId, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/peek`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
+  flushDoc(projectId, docId, callback) {
+    request.post(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/flush`,
+      (error, res, body) => callback(error, res, body)
+    )
+  },
+
+  setDocLines(projectId, docId, lines, source, userId, undoing, callback) {
+    request.post(
+      {
+        url: `http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
+        json: {
+          lines,
+          source,
+          user_id: userId,
+          undoing,
+        },
+      },
+      (error, res, body) => callback(error, res, body)
+    )
+  },
+
+  deleteDoc(projectId, docId, callback) {
+    request.del(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}`,
+      (error, res, body) => callback(error, res, body)
+    )
+  },
+
+  flushProject(projectId, callback) {
+    request.post(`http://127.0.0.1:3003/project/${projectId}/flush`, callback)
+  },
+
+  deleteProject(projectId, callback) {
+    request.del(`http://127.0.0.1:3003/project/${projectId}`, callback)
+  },
+
+  deleteProjectOnShutdown(projectId, callback) {
+    request.del(
+      `http://127.0.0.1:3003/project/${projectId}?background=true&shutdown=true`,
+      callback
+    )
+  },
+
+  flushOldProjects(callback) {
+    request.get(
+      'http://127.0.0.1:3003/flush_queued_projects?min_delete_age=1',
+      callback
+    )
+  },
+
+  acceptChange(projectId, docId, changeId, callback) {
+    request.post(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/${changeId}/accept`,
+      callback
+    )
+  },
+
+  acceptChanges(projectId, docId, changeIds, callback) {
+    request.post(
+      {
+        url: `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/change/accept`,
+        json: { change_ids: changeIds },
+      },
+      callback
+    )
+  },
+
+  removeComment(projectId, docId, comment, callback) {
+    request.del(
+      `http://127.0.0.1:3003/project/${projectId}/doc/${docId}/comment/${comment}`,
+      callback
+    )
+  },
+
+  getProjectDocs(projectId, projectStateHash, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/doc?state=${projectStateHash}`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
+  sendProjectUpdate(projectId, userId, updates, version, callback) {
+    request.post(
+      {
+        url: `http://127.0.0.1:3003/project/${projectId}`,
+        json: { userId, updates, version },
+      },
+      (error, res, body) => callback(error, res, body)
+    )
+  },
+}
--- a/services/document-updater/test/acceptance/js/helpers/MockDocstoreApi.js
+++ b/services/document-updater/test/acceptance/js/helpers/MockDocstoreApi.js
@@ -0,0 +1,111 @@
+const express = require('express')
+const bodyParser = require('body-parser')
+const app = express()
+const MAX_REQUEST_SIZE = 2 * (2 * 1024 * 1024 + 64 * 1024)
+
+const MockDocstoreApi = {
+  docs: {},
+
+  clearDocs() {
+    this.docs = {}
+  },
+
+  getDoc(projectId, docId) {
+    return this.docs[`${projectId}:${docId}`]
+  },
+
+  insertDoc(projectId, docId, doc) {
+    if (doc.version == null) {
+      doc.version = 0
+    }
+    if (doc.lines == null) {
+      doc.lines = []
+    }
+    this.docs[`${projectId}:${docId}`] = doc
+  },
+
+  patchDocument(projectId, docId, meta, callback) {
+    Object.assign(this.docs[`${projectId}:${docId}`], meta)
+    callback(null)
+  },
+
+  peekDocument(projectId, docId, callback) {
+    callback(null, this.docs[`${projectId}:${docId}`])
+  },
+
+  getAllDeletedDocs(projectId, callback) {
+    callback(
+      null,
+      Object.entries(this.docs)
+        .filter(([key, doc]) => key.startsWith(projectId) && doc.deleted)
+        .map(([key, doc]) => {
+          return {
+            _id: key.split(':')[1],
+            name: doc.name,
+            deletedAt: doc.deletedAt,
+          }
+        })
+    )
+  },
+
+  run() {
+    app.get('/project/:project_id/doc-deleted', (req, res, next) => {
+      this.getAllDeletedDocs(req.params.project_id, (error, docs) => {
+        if (error) {
+          res.sendStatus(500)
+        } else {
+          res.json(docs)
+        }
+      })
+    })
+
+    app.get('/project/:project_id/doc/:doc_id/peek', (req, res, next) => {
+      this.peekDocument(
+        req.params.project_id,
+        req.params.doc_id,
+        (error, doc) => {
+          if (error) {
+            res.sendStatus(500)
+          } else if (doc) {
+            res.json(doc)
+          } else {
+            res.sendStatus(404)
+          }
+        }
+      )
+    })
+
+    app.patch(
+      '/project/:project_id/doc/:doc_id',
+      bodyParser.json({ limit: MAX_REQUEST_SIZE }),
+      (req, res, next) => {
+        MockDocstoreApi.patchDocument(
+          req.params.project_id,
+          req.params.doc_id,
+          req.body,
+          error => {
+            if (error) {
+              res.sendStatus(500)
+            } else {
+              res.sendStatus(204)
+            }
+          }
+        )
+      }
+    )
+
+    app
+      .listen(3016, error => {
+        if (error) {
+          throw error
+        }
+      })
+      .on('error', error => {
+        console.error('error starting MockDocstoreApi:', error.message)
+        process.exit(1)
+      })
+  },
+}
+
+MockDocstoreApi.run()
+module.exports = MockDocstoreApi
--- a/services/document-updater/test/acceptance/js/helpers/MockProjectHistoryApi.js
+++ b/services/document-updater/test/acceptance/js/helpers/MockProjectHistoryApi.js
@@ -0,0 +1,40 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockProjectHistoryApi
+const express = require('express')
+const app = express()
+
+module.exports = MockProjectHistoryApi = {
+  flushProject(docId, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    return callback()
+  },
+
+  run() {
+    app.post('/project/:project_id/flush', (req, res, next) => {
+      return this.flushProject(req.params.project_id, error => {
+        if (error != null) {
+          return res.sendStatus(500)
+        } else {
+          return res.sendStatus(204)
+        }
+      })
+    })
+
+    return app.listen(3054, error => {
+      if (error != null) {
+        throw error
+      }
+    })
+  },
+}
+
+MockProjectHistoryApi.run()
--- a/services/document-updater/test/acceptance/js/helpers/MockWebApi.js
+++ b/services/document-updater/test/acceptance/js/helpers/MockWebApi.js
@@ -0,0 +1,121 @@
+/* eslint-disable
+    no-return-assign,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let MockWebApi
+const express = require('express')
+const bodyParser = require('body-parser')
+const app = express()
+const MAX_REQUEST_SIZE = 2 * (2 * 1024 * 1024 + 64 * 1024)
+
+module.exports = MockWebApi = {
+  docs: {},
+
+  clearDocs() {
+    return (this.docs = {})
+  },
+
+  insertDoc(projectId, docId, doc) {
+    if (doc.version == null) {
+      doc.version = 0
+    }
+    if (doc.lines == null) {
+      doc.lines = []
+    }
+    doc.pathname = '/a/b/c.tex'
+    return (this.docs[`${projectId}:${docId}`] = doc)
+  },
+
+  setDocument(
+    projectId,
+    docId,
+    lines,
+    version,
+    ranges,
+    lastUpdatedAt,
+    lastUpdatedBy,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    const doc =
+      this.docs[`${projectId}:${docId}`] ||
+      (this.docs[`${projectId}:${docId}`] = {})
+    doc.lines = lines
+    doc.version = version
+    doc.ranges = ranges
+    doc.pathname = '/a/b/c.tex'
+    doc.lastUpdatedAt = lastUpdatedAt
+    doc.lastUpdatedBy = lastUpdatedBy
+    return callback(null)
+  },
+
+  getDocument(projectId, docId, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    return callback(null, this.docs[`${projectId}:${docId}`])
+  },
+
+  run() {
+    app.get('/project/:project_id/doc/:doc_id', (req, res, next) => {
+      return this.getDocument(
+        req.params.project_id,
+        req.params.doc_id,
+        (error, doc) => {
+          if (error != null) {
+            return res.sendStatus(500)
+          } else if (doc != null) {
+            return res.send(JSON.stringify(doc))
+          } else {
+            return res.sendStatus(404)
+          }
+        }
+      )
+    })
+
+    app.post(
+      '/project/:project_id/doc/:doc_id',
+      bodyParser.json({ limit: MAX_REQUEST_SIZE }),
+      (req, res, next) => {
+        return MockWebApi.setDocument(
+          req.params.project_id,
+          req.params.doc_id,
+          req.body.lines,
+          req.body.version,
+          req.body.ranges,
+          req.body.lastUpdatedAt,
+          req.body.lastUpdatedBy,
+          error => {
+            if (error != null) {
+              return res.sendStatus(500)
+            } else {
+              return res.json({ rev: '123' })
+            }
+          }
+        )
+      }
+    )
+
+    return app
+      .listen(3000, error => {
+        if (error != null) {
+          throw error
+        }
+      })
+      .on('error', error => {
+        console.error('error starting MockWebApi:', error.message)
+        return process.exit(1)
+      })
+  },
+}
+
+MockWebApi.run()