first commit

services/web/modules/history-v1/index.mjs

@@ -0,0 +1,6 @@
+/** @import { WebModule } from "../../types/web-module" */
+
+/** @type {WebModule} */
+const HistoryModule = {}
+
+export default HistoryModule

services/web/modules/history-v1/test/acceptance/config/settings.test.js

@@ -0,0 +1,7 @@
+const base = require(process.env.BASE_CONFIG)
+
+module.exports = base.mergeWith({
+  test: {
+    counterInit: 190000,
+  },
+})

services/web/modules/history-v1/test/acceptance/src/HistoryTests.mjs

@@ -0,0 +1,338 @@
+import { expect } from 'chai'
+
+import _ from 'lodash'
+import { db, ObjectId } from '../../../../../app/src/infrastructure/mongodb.js'
+import User from '../../../../../test/acceptance/src/helpers/User.mjs'
+import MockV1HistoryApiClass from '../../../../../test/acceptance/src/mocks/MockV1HistoryApi.mjs'
+
+let MockV1HistoryApi
+
+before(function () {
+  MockV1HistoryApi = MockV1HistoryApiClass.instance()
+})
+
+describe('History', function () {
+  beforeEach(function (done) {
+    this.owner = new User()
+    this.owner.login(done)
+  })
+
+  describe('zip download of version', function () {
+    it('should stream the zip file of a version', function (done) {
+      this.owner.createProject('example-project', (error, projectId) => {
+        this.project_id = projectId
+        if (error) {
+          return done(error)
+        }
+        this.v1_history_id = 42
+        db.projects.updateOne(
+          {
+            _id: new ObjectId(this.project_id),
+          },
+          {
+            $set: {
+              'overleaf.history.id': this.v1_history_id,
+            },
+          },
+          error => {
+            if (error) {
+              return done(error)
+            }
+            this.owner.request(
+              `/project/${this.project_id}/version/42/zip`,
+              (error, response, body) => {
+                if (error) {
+                  return done(error)
+                }
+                expect(response.statusCode).to.equal(200)
+                expect(response.headers['content-type']).to.equal(
+                  'application/zip'
+                )
+                expect(response.headers['content-disposition']).to.equal(
+                  'attachment; filename="example-project (Version 42).zip"'
+                )
+                expect(body).to.equal(
+                  `Mock zip for ${this.v1_history_id} at version 42`
+                )
+                done()
+              }
+            )
+          }
+        )
+      })
+    })
+
+    describe('request abort', function () {
+      // Optional manual verification: add unique logging statements into
+      //   HistoryController._pipeHistoryZipToResponse
+      // in each of the `req.destroyed` branches and confirm that each branch
+      //  was covered.
+      beforeEach(function setupNewProject(done) {
+        this.owner.createProject('example-project', (error, projectId) => {
+          this.project_id = projectId
+          if (error) {
+            return done(error)
+          }
+          this.v1_history_id = 42
+          db.projects.updateOne(
+            { _id: new ObjectId(this.project_id) },
+            {
+              $set: {
+                'overleaf.history.id': this.v1_history_id,
+              },
+            },
+            done
+          )
+        })
+      })
+
+      it('should abort the upstream request', function (done) {
+        const request = this.owner.request(
+          `/project/${this.project_id}/version/100/zip`
+        )
+        request.on('error', err => {
+          if (err.code !== 'ECONNRESET') {
+            done(err)
+          }
+        })
+        request.on('response', response => {
+          expect(response.statusCode).to.equal(200)
+          let receivedChunks = 0
+          response.on('data', () => {
+            receivedChunks++
+          })
+          response.resume()
+
+          setTimeout(() => {
+            request.abort()
+            const receivedSoFar = receivedChunks
+            const sentSoFar = MockV1HistoryApi.sentChunks
+            // Ihe next assertions should verify that chunks are emitted
+            //  and received -- the exact number is not important.
+            // In theory we are now emitting the 3rd chunk,
+            //  so this should be exactly 3, to not make this
+            //  test flaky, we allow +- 2 chunks.
+            expect(sentSoFar).to.be.within(1, 4)
+            expect(receivedSoFar).to.be.within(1, 4)
+            setTimeout(() => {
+              // The fake-s3 service should have stopped emitting chunks.
+              // If not, that would be +5 in an ideal world (1 every 100ms).
+              // On the happy-path (it stopped) it emitted +1 which was
+              //  in-flight and another +1 before it received the abort.
+              expect(MockV1HistoryApi.sentChunks).to.be.below(sentSoFar + 5)
+              expect(MockV1HistoryApi.sentChunks).to.be.within(
+                sentSoFar,
+                sentSoFar + 2
+              )
+              done()
+            }, 500)
+          }, 200)
+        })
+      })
+
+      it('should skip the v1-history request', function (done) {
+        const request = this.owner.request(
+          `/project/${this.project_id}/version/100/zip`
+        )
+        setTimeout(() => {
+          // This is a race-condition to abort the request after the
+          //  processing of all the the express middleware completed.
+          // In case we abort before they complete, we do not hit our
+          //  abort logic, but express internal logic, which is OK.
+          request.abort()
+        }, 2)
+        request.on('error', done)
+        setTimeout(() => {
+          expect(MockV1HistoryApi.requestedZipPacks).to.equal(0)
+          done()
+        }, 500)
+      })
+
+      it('should skip the async-polling', function (done) {
+        const request = this.owner.request(
+          `/project/${this.project_id}/version/100/zip`
+        )
+        MockV1HistoryApi.events.on('v1-history-pack-zip', () => {
+          request.abort()
+        })
+        request.on('error', done)
+        setTimeout(() => {
+          expect(MockV1HistoryApi.fakeZipCall).to.equal(0)
+          done()
+        }, 3000) // initial polling delay is 2s
+      })
+
+      it('should skip the upstream request', function (done) {
+        const request = this.owner.request(
+          `/project/${this.project_id}/version/100/zip`
+        )
+        MockV1HistoryApi.events.on('v1-history-pack-zip', () => {
+          setTimeout(() => {
+            request.abort()
+          }, 1000)
+        })
+        request.on('error', done)
+        setTimeout(() => {
+          expect(MockV1HistoryApi.fakeZipCall).to.equal(0)
+          done()
+        }, 3000) // initial polling delay is 2s
+      })
+    })
+
+    it('should return 402 for non-v2-history project', function (done) {
+      this.owner.createProject('non-v2-project', (error, projectId) => {
+        this.project_id = projectId
+        if (error) {
+          return done(error)
+        }
+        db.projects.updateOne(
+          {
+            _id: new ObjectId(this.project_id),
+          },
+          {
+            $unset: {
+              'overleaf.history.id': true,
+            },
+          },
+          error => {
+            if (error) {
+              return done(error)
+            }
+            this.owner.request(
+              `/project/${this.project_id}/version/42/zip`,
+              (error, response, body) => {
+                if (error) {
+                  return done(error)
+                }
+                expect(response.statusCode).to.equal(402)
+                done()
+              }
+            )
+          }
+        )
+      })
+    })
+  })
+
+  describe('zip download, with upstream 404', function () {
+    beforeEach(function () {
+      _.remove(
+        MockV1HistoryApi.app._router.stack,
+        appRoute =>
+          appRoute.route?.path ===
+          '/api/projects/:project_id/version/:version/zip'
+      )
+      MockV1HistoryApi.app.post(
+        '/api/projects/:project_id/version/:version/zip',
+        (req, res, next) => {
+          res.sendStatus(404)
+        }
+      )
+    })
+
+    afterEach(function () {
+      MockV1HistoryApi = MockV1HistoryApiClass.instance()
+      MockV1HistoryApi.reset()
+      MockV1HistoryApi.applyRoutes()
+    })
+
+    it('should produce 404 when post request produces 404', function (done) {
+      this.owner.createProject('example-project', (error, projectId) => {
+        if (error) {
+          return done(error)
+        }
+        this.project_id = projectId
+        this.v1_history_id = 42
+        db.projects.updateOne(
+          {
+            _id: new ObjectId(this.project_id),
+          },
+          {
+            $set: {
+              'overleaf.history.id': this.v1_history_id,
+            },
+          },
+          error => {
+            if (error) {
+              return done(error)
+            }
+            this.owner.request(
+              `/project/${this.project_id}/version/42/zip`,
+              (error, response, body) => {
+                if (error) {
+                  return done(error)
+                }
+                expect(response.statusCode).to.equal(404)
+                done()
+              }
+            )
+          }
+        )
+      })
+    })
+  })
+
+  describe('zip download, with no zipUrl from upstream', function () {
+    beforeEach(function () {
+      _.remove(
+        MockV1HistoryApi.app._router.stack,
+        appRoute =>
+          appRoute.route?.path ===
+          '/api/projects/:project_id/version/:version/zip'
+      )
+      MockV1HistoryApi.app.get(
+        '/api/projects/:project_id/version/:version/zip',
+        (req, res, next) => {
+          res.sendStatus(500)
+        }
+      )
+      MockV1HistoryApi.app.post(
+        '/api/projects/:project_id/version/:version/zip',
+        (req, res, next) => {
+          res.json({ message: 'lol' })
+        }
+      )
+    })
+
+    afterEach(function () {
+      MockV1HistoryApi = MockV1HistoryApiClass.instance()
+      MockV1HistoryApi.reset()
+      MockV1HistoryApi.applyRoutes()
+    })
+
+    it('should produce 500', function (done) {
+      this.owner.createProject('example-project', (error, projectId) => {
+        if (error) {
+          return done(error)
+        }
+        this.project_id = projectId
+        this.v1_history_id = 42
+        db.projects.updateOne(
+          {
+            _id: new ObjectId(this.project_id),
+          },
+          {
+            $set: {
+              'overleaf.history.id': this.v1_history_id,
+            },
+          },
+          error => {
+            if (error) {
+              return done(error)
+            }
+            this.owner.request(
+              `/project/${this.project_id}/version/42/zip`,
+              (error, response, body) => {
+                if (error) {
+                  return done(error)
+                }
+                expect(response.statusCode).to.equal(500)
+                done()
+              }
+            )
+          }
+        )
+      })
+    })
+  })
+})

services/web/modules/history-v1/test/acceptance/src/Init.mjs

@@ -0,0 +1,22 @@
+import '../../../../../test/acceptance/src/helpers/InitApp.mjs'
+import MockDocstoreApi from '../../../../../test/acceptance/src/mocks/MockDocstoreApi.mjs'
+import MockDocUpdaterApi from '../../../../../test/acceptance/src/mocks/MockDocUpdaterApi.mjs'
+import MockFilestoreApi from '../../../../../test/acceptance/src/mocks/MockFilestoreApi.mjs'
+import MockNotificationsApi from '../../../../../test/acceptance/src/mocks/MockNotificationsApi.mjs'
+import MockProjectHistoryApi from '../../../../../test/acceptance/src/mocks/MockProjectHistoryApi.mjs'
+import MockSpellingApi from '../../../../../test/acceptance/src/mocks/MockSpellingApi.mjs'
+import MockV1Api from '../../../../../test/acceptance/src/mocks/MockV1Api.mjs'
+import MockV1HistoryApi from '../../../../../test/acceptance/src/mocks/MockV1HistoryApi.mjs'
+
+const mockOpts = {
+  debug: ['1', 'true', 'TRUE'].includes(process.env.DEBUG_MOCKS),
+}
+
+MockDocstoreApi.initialize(23016, mockOpts)
+MockDocUpdaterApi.initialize(23003, mockOpts)
+MockFilestoreApi.initialize(23009, mockOpts)
+MockNotificationsApi.initialize(23042, mockOpts)
+MockProjectHistoryApi.initialize(23054, mockOpts)
+MockSpellingApi.initialize(23005, mockOpts)
+MockV1Api.initialize(25000, mockOpts)
+MockV1HistoryApi.initialize(23100, mockOpts)

services/web/modules/history-v1/test/acceptance/src/LabelsTests.mjs

@@ -0,0 +1,114 @@
+import { expect } from 'chai'
+import mongodb from 'mongodb-legacy'
+import User from '../../../../../test/acceptance/src/helpers/User.mjs'
+import MockProjectHistoryApiClass from '../../../../../test/acceptance/src/mocks/MockProjectHistoryApi.mjs'
+
+const { ObjectId } = mongodb
+
+let MockProjectHistoryApi
+
+before(function () {
+  MockProjectHistoryApi = MockProjectHistoryApiClass.instance()
+})
+
+describe('Labels', function () {
+  beforeEach(function (done) {
+    this.owner = new User()
+    this.owner.login(error => {
+      if (error) {
+        throw error
+      }
+      this.owner.createProject(
+        'example-project',
+        { template: 'example' },
+        (error, projectId) => {
+          this.project_id = projectId
+          if (error) {
+            throw error
+          }
+          done()
+        }
+      )
+    })
+  })
+
+  it('getting labels', function (done) {
+    const labelId = new ObjectId().toString()
+    const comment = 'a label comment'
+    const version = 3
+    MockProjectHistoryApi.addLabel(this.project_id, {
+      id: labelId,
+      comment,
+      version,
+    })
+
+    this.owner.request(
+      {
+        method: 'GET',
+        url: `/project/${this.project_id}/labels`,
+        json: true,
+      },
+      (error, response, body) => {
+        if (error) {
+          throw error
+        }
+        expect(response.statusCode).to.equal(200)
+        expect(body).to.deep.equal([{ id: labelId, comment, version }])
+        done()
+      }
+    )
+  })
+
+  it('creating a label', function (done) {
+    const comment = 'a label comment'
+    const version = 3
+
+    this.owner.request(
+      {
+        method: 'POST',
+        url: `/project/${this.project_id}/labels`,
+        json: { comment, version },
+      },
+      (error, response, body) => {
+        if (error) {
+          throw error
+        }
+        expect(response.statusCode).to.equal(200)
+        const { label_id: labelId } = body
+        expect(MockProjectHistoryApi.getLabels(this.project_id)).to.deep.equal([
+          { id: labelId, comment, version },
+        ])
+        done()
+      }
+    )
+  })
+
+  it('deleting a label', function (done) {
+    const labelId = new ObjectId().toString()
+    const comment = 'a label comment'
+    const version = 3
+    MockProjectHistoryApi.addLabel(this.project_id, {
+      id: labelId,
+      comment,
+      version,
+    })
+
+    this.owner.request(
+      {
+        method: 'DELETE',
+        url: `/project/${this.project_id}/labels/${labelId}`,
+        json: true,
+      },
+      (error, response, body) => {
+        if (error) {
+          throw error
+        }
+        expect(response.statusCode).to.equal(204)
+        expect(MockProjectHistoryApi.getLabels(this.project_id)).to.deep.equal(
+          []
+        )
+        done()
+      }
+    )
+  })
+})

services/web/modules/history-v1/test/acceptance/src/RestoringFilesTest.mjs

@@ -0,0 +1,307 @@
+import { expect } from 'chai'
+
+import _ from 'lodash'
+import fs from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import Path from 'node:path'
+import User from '../../../../../test/acceptance/src/helpers/User.mjs'
+import MockProjectHistoryApiClass from '../../../../../test/acceptance/src/mocks/MockProjectHistoryApi.mjs'
+import MockDocstoreApiClass from '../../../../../test/acceptance/src/mocks/MockDocstoreApi.mjs'
+import MockFilestoreApiClass from '../../../../../test/acceptance/src/mocks/MockFilestoreApi.mjs'
+import MockV1HistoryApiClass from '../../../../../test/acceptance/src/mocks/MockV1HistoryApi.mjs'
+import Features from '../../../../../app/src/infrastructure/Features.js'
+
+let MockProjectHistoryApi, MockDocstoreApi, MockFilestoreApi, MockV1HistoryApi
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+
+before(function () {
+  MockProjectHistoryApi = MockProjectHistoryApiClass.instance()
+  MockDocstoreApi = MockDocstoreApiClass.instance()
+  MockFilestoreApi = MockFilestoreApiClass.instance()
+  MockV1HistoryApi = MockV1HistoryApiClass.instance()
+})
+
+describe('RestoringFiles', function () {
+  beforeEach(function (done) {
+    this.owner = new User()
+    this.owner.login(error => {
+      if (error) {
+        throw error
+      }
+      this.owner.createProject(
+        'example-project',
+        { template: 'example' },
+        (error, projectId) => {
+          this.project_id = projectId
+          if (error) {
+            throw error
+          }
+          done()
+        }
+      )
+    })
+  })
+
+  describe('restoring from v2 history', function () {
+    describe('restoring a text file', function () {
+      beforeEach(function (done) {
+        MockProjectHistoryApi.addOldFile(
+          this.project_id,
+          42,
+          'foo.tex',
+          'hello world, this is foo.tex!'
+        )
+        this.owner.request(
+          {
+            method: 'POST',
+            url: `/project/${this.project_id}/restore_file`,
+            json: {
+              pathname: 'foo.tex',
+              version: 42,
+            },
+          },
+          (error, response, body) => {
+            if (error) {
+              throw error
+            }
+            expect(response.statusCode).to.equal(200)
+            done()
+          }
+        )
+      })
+
+      it('should have created a doc', function (done) {
+        this.owner.getProject(this.project_id, (error, project) => {
+          if (error) {
+            throw error
+          }
+          let doc = _.find(
+            project.rootFolder[0].docs,
+            doc => doc.name === 'foo.tex'
+          )
+          doc = MockDocstoreApi.docs[this.project_id][doc._id]
+          expect(doc.lines).to.deep.equal(['hello world, this is foo.tex!'])
+          done()
+        })
+      })
+    })
+
+    describe('restoring a binary file', function () {
+      beforeEach(function (done) {
+        this.pngData = fs.readFileSync(
+          Path.resolve(
+            __dirname,
+            '../../../../../test/acceptance/files/1pixel.png'
+          ),
+          'binary'
+        )
+        MockProjectHistoryApi.addOldFile(
+          this.project_id,
+          42,
+          'image.png',
+          this.pngData
+        )
+        this.owner.request(
+          {
+            method: 'POST',
+            url: `/project/${this.project_id}/restore_file`,
+            json: {
+              pathname: 'image.png',
+              version: 42,
+            },
+          },
+          (error, response, body) => {
+            if (error) {
+              throw error
+            }
+            expect(response.statusCode).to.equal(200)
+            done()
+          }
+        )
+      })
+
+      if (Features.hasFeature('project-history-blobs')) {
+        it('should have created a file in history-v1', function (done) {
+          this.owner.getProject(this.project_id, (error, project) => {
+            if (error) {
+              throw error
+            }
+            let file = _.find(
+              project.rootFolder[0].fileRefs,
+              file => file.name === 'image.png'
+            )
+            file =
+              MockV1HistoryApi.blobs[project.overleaf.history.id.toString()][
+                file.hash
+              ]
+            expect(file).to.deep.equal(Buffer.from(this.pngData))
+            done()
+          })
+        })
+      }
+      if (Features.hasFeature('filestore')) {
+        it('should have created a file in filestore', function (done) {
+          this.owner.getProject(this.project_id, (error, project) => {
+            if (error) {
+              throw error
+            }
+            let file = _.find(
+              project.rootFolder[0].fileRefs,
+              file => file.name === 'image.png'
+            )
+            file = MockFilestoreApi.getFile(this.project_id, file._id)
+            expect(file).to.deep.equal(this.pngData)
+            done()
+          })
+        })
+      }
+    })
+
+    describe('restoring to a directory that exists', function () {
+      beforeEach(function (done) {
+        MockProjectHistoryApi.addOldFile(
+          this.project_id,
+          42,
+          'foldername/foo2.tex',
+          'hello world, this is foo-2.tex!'
+        )
+        this.owner.request.post(
+          {
+            uri: `project/${this.project_id}/folder`,
+            json: {
+              name: 'foldername',
+            },
+          },
+          (error, response, body) => {
+            if (error) {
+              throw error
+            }
+            expect(response.statusCode).to.equal(200)
+            this.owner.request(
+              {
+                method: 'POST',
+                url: `/project/${this.project_id}/restore_file`,
+                json: {
+                  pathname: 'foldername/foo2.tex',
+                  version: 42,
+                },
+              },
+              (error, response, body) => {
+                if (error) {
+                  throw error
+                }
+                expect(response.statusCode).to.equal(200)
+                done()
+              }
+            )
+          }
+        )
+      })
+
+      it('should have created the doc in the named folder', function (done) {
+        this.owner.getProject(this.project_id, (error, project) => {
+          if (error) {
+            throw error
+          }
+          const folder = _.find(
+            project.rootFolder[0].folders,
+            folder => folder.name === 'foldername'
+          )
+          let doc = _.find(folder.docs, doc => doc.name === 'foo2.tex')
+          doc = MockDocstoreApi.docs[this.project_id][doc._id]
+          expect(doc.lines).to.deep.equal(['hello world, this is foo-2.tex!'])
+          done()
+        })
+      })
+    })
+
+    describe('restoring to a directory that no longer exists', function () {
+      beforeEach(function (done) {
+        MockProjectHistoryApi.addOldFile(
+          this.project_id,
+          42,
+          'nothere/foo3.tex',
+          'hello world, this is foo-3.tex!'
+        )
+        this.owner.request(
+          {
+            method: 'POST',
+            url: `/project/${this.project_id}/restore_file`,
+            json: {
+              pathname: 'nothere/foo3.tex',
+              version: 42,
+            },
+          },
+          (error, response, body) => {
+            if (error) {
+              throw error
+            }
+            expect(response.statusCode).to.equal(200)
+            done()
+          }
+        )
+      })
+
+      it('should have created the folder and restored the doc to it', function (done) {
+        this.owner.getProject(this.project_id, (error, project) => {
+          if (error) {
+            throw error
+          }
+          const folder = _.find(
+            project.rootFolder[0].folders,
+            folder => folder.name === 'nothere'
+          )
+          expect(folder).to.exist
+          let doc = _.find(folder.docs, doc => doc.name === 'foo3.tex')
+          doc = MockDocstoreApi.docs[this.project_id][doc._id]
+          expect(doc.lines).to.deep.equal(['hello world, this is foo-3.tex!'])
+          done()
+        })
+      })
+    })
+
+    describe('restoring to a filename that already exists', function () {
+      beforeEach(function (done) {
+        MockProjectHistoryApi.addOldFile(
+          this.project_id,
+          42,
+          'main.tex',
+          'hello world, this is main.tex!'
+        )
+        this.owner.request(
+          {
+            method: 'POST',
+            url: `/project/${this.project_id}/restore_file`,
+            json: {
+              pathname: 'main.tex',
+              version: 42,
+            },
+          },
+          (error, response, body) => {
+            if (error) {
+              throw error
+            }
+            expect(response.statusCode).to.equal(200)
+            done()
+          }
+        )
+      })
+
+      it('should have created the doc in the root folder', function (done) {
+        this.owner.getProject(this.project_id, (error, project) => {
+          if (error) {
+            throw error
+          }
+          let doc = _.find(project.rootFolder[0].docs, doc =>
+            doc.name.match(/main \(Restored on/)
+          )
+          expect(doc).to.exist
+          doc = MockDocstoreApi.docs[this.project_id][doc._id]
+          expect(doc.lines).to.deep.equal(['hello world, this is main.tex!'])
+          done()
+        })
+      })
+    })
+  })
+})

services/web/modules/history-v1/test/acceptance/tsconfig.json

@@ -0,0 +1 @@
+{ "extends": "../../../../tsconfig.backend.json" }