first commit
This commit is contained in:
@@ -0,0 +1,556 @@
|
||||
import { expect } from 'chai'
|
||||
import sinon from 'sinon'
|
||||
import { strict as esmock } from 'esmock'
|
||||
|
||||
const MODULE_PATH = '../../../../app/js/RedisManager.js'
|
||||
|
||||
describe('RedisManager', function () {
|
||||
beforeEach(async function () {
|
||||
this.rclient = new FakeRedis()
|
||||
this.RedisWrapper = {
|
||||
createClient: sinon.stub().returns(this.rclient),
|
||||
}
|
||||
this.Settings = {
|
||||
redis: {
|
||||
project_history: {
|
||||
key_schema: {
|
||||
projectHistoryOps({ project_id: projectId }) {
|
||||
return `Project:HistoryOps:{${projectId}}`
|
||||
},
|
||||
projectHistoryFirstOpTimestamp({ project_id: projectId }) {
|
||||
return `ProjectHistory:FirstOpTimestamp:{${projectId}}`
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
this.Metrics = {
|
||||
timing: sinon.stub(),
|
||||
summary: sinon.stub(),
|
||||
globalGauge: sinon.stub(),
|
||||
}
|
||||
this.RedisManager = await esmock(MODULE_PATH, {
|
||||
'@overleaf/redis-wrapper': this.RedisWrapper,
|
||||
'@overleaf/settings': this.Settings,
|
||||
'@overleaf/metrics': this.Metrics,
|
||||
})
|
||||
|
||||
this.projectId = 'project-id-123'
|
||||
this.batchSize = 100
|
||||
this.historyOpsKey = `Project:HistoryOps:{${this.projectId}}`
|
||||
this.firstOpTimestampKey = `ProjectHistory:FirstOpTimestamp:{${this.projectId}}`
|
||||
|
||||
this.updates = [
|
||||
{ v: 42, op: ['a', 'b', 'c', 'd'] },
|
||||
{ v: 45, op: ['e', 'f', 'g', 'h'] },
|
||||
]
|
||||
this.extraUpdates = [{ v: 100, op: ['i', 'j', 'k'] }]
|
||||
this.rawUpdates = this.updates.map(update => JSON.stringify(update))
|
||||
this.extraRawUpdates = this.extraUpdates.map(update =>
|
||||
JSON.stringify(update)
|
||||
)
|
||||
})
|
||||
|
||||
describe('getRawUpdatesBatch', function () {
|
||||
it('gets a small number of updates in one batch', async function () {
|
||||
const updates = makeUpdates(2)
|
||||
const rawUpdates = makeRawUpdates(updates)
|
||||
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
||||
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
||||
this.projectId,
|
||||
100
|
||||
)
|
||||
expect(result).to.deep.equal({ rawUpdates, hasMore: false })
|
||||
})
|
||||
|
||||
it('gets a larger number of updates in several batches', async function () {
|
||||
const updates = makeUpdates(
|
||||
this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2 + 12
|
||||
)
|
||||
const rawUpdates = makeRawUpdates(updates)
|
||||
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
||||
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
||||
this.projectId,
|
||||
5000
|
||||
)
|
||||
expect(result).to.deep.equal({ rawUpdates, hasMore: false })
|
||||
})
|
||||
|
||||
it("doesn't return more than the number of updates requested", async function () {
|
||||
const updates = makeUpdates(100)
|
||||
const rawUpdates = makeRawUpdates(updates)
|
||||
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
||||
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
||||
this.projectId,
|
||||
75
|
||||
)
|
||||
expect(result).to.deep.equal({
|
||||
rawUpdates: rawUpdates.slice(0, 75),
|
||||
hasMore: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseDocUpdates', function () {
|
||||
it('should return the parsed ops', function () {
|
||||
const updates = makeUpdates(12)
|
||||
const rawUpdates = makeRawUpdates(updates)
|
||||
this.RedisManager.parseDocUpdates(rawUpdates).should.deep.equal(updates)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getUpdatesInBatches', function () {
|
||||
beforeEach(function () {
|
||||
this.runner = sinon.stub().resolves()
|
||||
})
|
||||
|
||||
describe('single batch smaller than batch size', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(2)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
3,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner once', function () {
|
||||
this.runner.callCount.should.equal(1)
|
||||
})
|
||||
|
||||
it('calls the runner with the updates', function () {
|
||||
this.runner.should.have.been.calledWith(this.updates)
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
|
||||
it('deletes the first op timestamp', function () {
|
||||
expect(this.rclient.del).to.have.been.calledWith(
|
||||
this.firstOpTimestampKey
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('single batch at batch size', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(123)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner once', function () {
|
||||
this.runner.callCount.should.equal(1)
|
||||
})
|
||||
|
||||
it('calls the runner with the updates', function () {
|
||||
this.runner.should.have.been.calledWith(this.updates)
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
|
||||
it('deletes the first op timestamp', function () {
|
||||
expect(this.rclient.del).to.have.been.calledWith(
|
||||
this.firstOpTimestampKey
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('single batch exceeding size limit on updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(2, [
|
||||
'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD),
|
||||
])
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the first update', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, 1))
|
||||
})
|
||||
|
||||
it('calls the runner with the second update', function () {
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(this.updates.slice(1))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('two batches with first update below and second update above the size limit on updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(2, [
|
||||
'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD / 2),
|
||||
])
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the first update', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, 1))
|
||||
})
|
||||
|
||||
it('calls the runner with the second update', function () {
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(this.updates.slice(1))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('single batch exceeding op count limit on updates', function () {
|
||||
beforeEach(async function () {
|
||||
const ops = Array(this.RedisManager.MAX_UPDATE_OP_LENGTH + 1).fill('op')
|
||||
this.updates = makeUpdates(2, { op: ops })
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the first update', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, 1))
|
||||
})
|
||||
|
||||
it('calls the runner with the second update', function () {
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(this.updates.slice(1))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('single batch exceeding doc content count', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(
|
||||
this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT + 3,
|
||||
{ resyncDocContent: 123 }
|
||||
)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the first batch of updates', function () {
|
||||
this.runner.should.have.been.calledWith(
|
||||
this.updates.slice(0, this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner with the second batch of updates', function () {
|
||||
this.runner.should.have.been.calledWith(
|
||||
this.updates.slice(this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
|
||||
)
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('two batches with first update below and second update above the ops length limit on updates', function () {
|
||||
beforeEach(async function () {
|
||||
// set the threshold below the size of the first update
|
||||
this.updates = makeUpdates(2, { op: ['op1', 'op2'] })
|
||||
this.updates[1].op = Array(
|
||||
this.RedisManager.MAX_UPDATE_OP_LENGTH + 2
|
||||
).fill('op')
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
123,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the first update', function () {
|
||||
this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
|
||||
})
|
||||
|
||||
it('calls the runner with the second update', function () {
|
||||
this.runner.should.have.been.calledWith(this.updates.slice(1))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('two batches, one partial', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(15)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
10,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the updates', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, 10))
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(this.updates.slice(10))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('two full batches', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(20)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
10,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(2)
|
||||
})
|
||||
|
||||
it('calls the runner with the updates', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, 10))
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(this.updates.slice(10))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('three full bathches, bigger than the Redis read batch size', function () {
|
||||
beforeEach(async function () {
|
||||
this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2
|
||||
this.updates = makeUpdates(this.batchSize * 3)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
await this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
this.batchSize,
|
||||
this.runner
|
||||
)
|
||||
})
|
||||
|
||||
it('calls the runner twice', function () {
|
||||
this.runner.callCount.should.equal(3)
|
||||
})
|
||||
|
||||
it('calls the runner with the updates', function () {
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, this.batchSize))
|
||||
this.runner
|
||||
.getCall(1)
|
||||
.should.have.been.calledWith(
|
||||
this.updates.slice(this.batchSize, this.batchSize * 2)
|
||||
)
|
||||
this.runner
|
||||
.getCall(2)
|
||||
.should.have.been.calledWith(this.updates.slice(this.batchSize * 2))
|
||||
})
|
||||
|
||||
it('deletes the applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('error when first reading updates', function () {
|
||||
beforeEach(async function () {
|
||||
this.updates = makeUpdates(10)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
this.rclient.throwErrorOnLrangeCall(0)
|
||||
await expect(
|
||||
this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
2,
|
||||
this.runner
|
||||
)
|
||||
).to.be.rejected
|
||||
})
|
||||
|
||||
it('does not delete any updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
|
||||
this.rawUpdates
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('error when reading updates for a second batch', function () {
|
||||
beforeEach(async function () {
|
||||
this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE - 1
|
||||
this.updates = makeUpdates(this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2)
|
||||
this.rawUpdates = makeRawUpdates(this.updates)
|
||||
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
||||
this.rclient.throwErrorOnLrangeCall(1)
|
||||
await expect(
|
||||
this.RedisManager.promises.getUpdatesInBatches(
|
||||
this.projectId,
|
||||
this.batchSize,
|
||||
this.runner
|
||||
)
|
||||
).to.be.rejected
|
||||
})
|
||||
|
||||
it('calls the runner with the first batch of updates', function () {
|
||||
this.runner.should.have.been.calledOnce
|
||||
this.runner
|
||||
.getCall(0)
|
||||
.should.have.been.calledWith(this.updates.slice(0, this.batchSize))
|
||||
})
|
||||
|
||||
it('deletes only the first batch of applied updates', function () {
|
||||
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
|
||||
this.rawUpdates.slice(this.batchSize)
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
class FakeRedis {
|
||||
constructor() {
|
||||
this.data = new Map()
|
||||
this.del = sinon.stub()
|
||||
this.lrangeCallCount = -1
|
||||
}
|
||||
|
||||
setList(key, list) {
|
||||
this.data.set(key, list)
|
||||
}
|
||||
|
||||
getList(key) {
|
||||
return this.data.get(key)
|
||||
}
|
||||
|
||||
throwErrorOnLrangeCall(callNum) {
|
||||
this.lrangeCallThrowingError = callNum
|
||||
}
|
||||
|
||||
async lrange(key, start, stop) {
|
||||
this.lrangeCallCount += 1
|
||||
if (
|
||||
this.lrangeCallThrowingError != null &&
|
||||
this.lrangeCallThrowingError === this.lrangeCallCount
|
||||
) {
|
||||
throw new Error('LRANGE failed!')
|
||||
}
|
||||
const list = this.data.get(key) ?? []
|
||||
return list.slice(start, stop + 1)
|
||||
}
|
||||
|
||||
async lrem(key, count, elementToRemove) {
|
||||
expect(count).to.be.greaterThan(0)
|
||||
const original = this.data.get(key) ?? []
|
||||
const filtered = original.filter(element => {
|
||||
if (count > 0 && element === elementToRemove) {
|
||||
count--
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
this.data.set(key, filtered)
|
||||
}
|
||||
|
||||
async exec() {
|
||||
// Nothing to do
|
||||
}
|
||||
|
||||
multi() {
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
function makeUpdates(updateCount, extraFields = {}) {
|
||||
const updates = []
|
||||
for (let i = 0; i < updateCount; i++) {
|
||||
updates.push({ v: i, ...extraFields })
|
||||
}
|
||||
return updates
|
||||
}
|
||||
|
||||
function makeRawUpdates(updates) {
|
||||
return updates.map(JSON.stringify)
|
||||
}
|
||||
Reference in New Issue
Block a user