first commit

This commit is contained in:
2025-04-24 13:11:28 +08:00
commit ff9c54d5e4
5960 changed files with 834111 additions and 0 deletions

View File

@@ -0,0 +1,531 @@
import {
ConnectionError,
ConnectionState,
ExternalHeartbeat,
SocketDebuggingInfo,
} from './types/connection-state'
import SocketIoShim from '../../../ide/connection/SocketIoShim'
import getMeta from '../../../utils/meta'
import { Socket } from '@/features/ide-react/connection/types/socket'
import { debugConsole } from '@/utils/debugging'
import { isSplitTestEnabled } from '@/utils/splitTestUtils'
const ONE_HOUR_IN_MS = 1000 * 60 * 60
const TWO_MINUTES_IN_MS = 2 * 60 * 1000
const DISCONNECT_AFTER_MS = ONE_HOUR_IN_MS * 24
const CONNECTION_ERROR_RECONNECT_DELAY = 1000
const USER_ACTIVITY_RECONNECT_NOW_DELAY = 1000
const USER_ACTIVITY_RECONNECT_DELAY = 5000
const JOIN_PROJECT_RATE_LIMITED_DELAY = 15 * 1000
const BACK_OFF_RECONNECT_OFFLINE = 5000
const RECONNECT_GRACEFULLY_RETRY_INTERVAL_MS = 5000
const MAX_RECONNECT_GRACEFULLY_INTERVAL_MS = getMeta(
'ol-maxReconnectGracefullyIntervalMs'
)
const MAX_RETRY_CONNECT = 5
const RETRY_WEBSOCKET = 3
const externalSocketHeartbeat = isSplitTestEnabled('external-socket-heartbeat')
const initialState: ConnectionState = {
readyState: WebSocket.CLOSED,
forceDisconnected: false,
inactiveDisconnect: false,
lastConnectionAttempt: 0,
reconnectAt: null,
forcedDisconnectDelay: 0,
error: '',
}
export class StateChangeEvent extends CustomEvent<{
state: ConnectionState
previousState: ConnectionState
}> {}
export class ConnectionManager extends EventTarget {
state: ConnectionState = initialState
private connectionAttempt: number | null = null
private gracefullyReconnectUntil = 0
private lastUserActivity: number
private protocolVersion = -1
private readonly idleDisconnectInterval: number
private reconnectCountdownInterval = 0
private websocketFailureCount = 0
readonly socket: Socket
private userIsLeavingPage = false
private externalHeartbeatInterval?: number
private externalHeartbeat: ExternalHeartbeat = {
currentStart: 0,
lastSuccess: 0,
lastLatency: 0,
}
constructor() {
super()
this.lastUserActivity = performance.now()
this.idleDisconnectInterval = window.setInterval(() => {
this.disconnectIfIdleSince(DISCONNECT_AFTER_MS)
}, ONE_HOUR_IN_MS)
window.addEventListener('online', () => this.onOnline())
window.addEventListener('beforeunload', () => {
this.userIsLeavingPage = true
if (this.socket.socket.transport?.name === 'xhr-polling') {
// Websockets will close automatically.
this.socket.socket.disconnect()
}
})
const parsedURL = new URL(
getMeta('ol-wsUrl') || '/socket.io',
window.origin
)
const query = new URLSearchParams({
projectId: getMeta('ol-project_id'),
})
if (externalSocketHeartbeat) {
query.set('esh', '1')
query.set('ssp', '1') // with server-side ping
}
const socket = SocketIoShim.connect(parsedURL.origin, {
resource: parsedURL.pathname.slice(1),
'auto connect': false,
'connect timeout': 30 * 1000,
'force new connection': true,
query: query.toString(),
reconnect: false,
}) as unknown as Socket
this.socket = socket
// bail out if socket.io failed to load (e.g. the real-time server is down)
if (typeof window.io !== 'object') {
this.switchToWsFallbackIfPossible()
debugConsole.error(
'Socket.io javascript not loaded. Please check that the real-time service is running and accessible.'
)
this.changeState({
...this.state,
error: 'io-not-loaded',
})
return
}
socket.on('connect', () => this.onConnect())
socket.on('disconnect', () => this.onDisconnect())
socket.on('error', err => this.onConnectError(err))
socket.on('connect_failed', err => this.onConnectError(err))
socket.on('joinProjectResponse', body => this.onJoinProjectResponse(body))
socket.on('connectionRejected', err => this.onConnectionRejected(err))
socket.on('reconnectGracefully', () => this.onReconnectGracefully())
socket.on('forceDisconnect', (_, delay) => this.onForceDisconnect(delay))
socket.on(
'serverPing',
(counter, timestamp, serverTransport, serverSessionId) =>
this.sendPingResponse(
counter,
timestamp,
serverTransport,
serverSessionId
)
)
this.tryReconnect()
}
close(error: ConnectionError) {
this.onForceDisconnect(0, error)
}
tryReconnectNow() {
this.tryReconnectWithBackoff(USER_ACTIVITY_RECONNECT_NOW_DELAY)
}
// Called when document is clicked or the editor cursor changes
registerUserActivity() {
this.lastUserActivity = performance.now()
this.userIsLeavingPage = false
this.ensureIsConnected()
}
getSocketDebuggingInfo(): SocketDebuggingInfo {
return {
client_id: this.socket.socket?.sessionid,
transport: this.socket.socket?.transport?.name,
publicId: this.socket.publicId,
lastUserActivity: this.lastUserActivity,
connectionState: this.state,
externalHeartbeat: this.externalHeartbeat,
}
}
private changeState(state: ConnectionState) {
const previousState = this.state
this.state = state
debugConsole.log('[ConnectionManager] changed state', {
previousState,
state,
})
this.dispatchEvent(
new StateChangeEvent('statechange', { detail: { state, previousState } })
)
}
private switchToWsFallbackIfPossible() {
const search = new URLSearchParams(window.location.search)
if (getMeta('ol-wsUrl') && search.get('ws') !== 'fallback') {
// if we tried to boot from a custom real-time backend and failed,
// try reloading and falling back to the siteUrl
search.set('ws', 'fallback')
window.location.search = search.toString()
return true
}
return false
}
private onOnline() {
if (!this.state.inactiveDisconnect) this.ensureIsConnected()
}
private onConnectionRejected(err: any) {
switch (err?.message) {
case 'retry': // pending real-time shutdown
this.startAutoReconnectCountdown(0)
break
case 'rate-limit hit when joining project': // rate-limited
this.changeState({
...this.state,
error: 'rate-limited',
})
break
case 'not authorized': // not logged in
case 'invalid session': // expired session
this.changeState({
...this.state,
error: 'not-logged-in',
forceDisconnected: true,
})
break
case 'project not found': // project has been deleted
this.changeState({
...this.state,
error: 'project-deleted',
forceDisconnected: true,
})
break
default:
this.changeState({
...this.state,
error: 'unable-to-join',
})
break
}
}
private onConnectError(err: any) {
if (
this.socket.socket.transport?.name === 'websocket' &&
err instanceof Event &&
err.target instanceof WebSocket
) {
this.websocketFailureCount++
}
if (this.connectionAttempt === null) return // ignore errors once connected.
if (this.connectionAttempt++ < MAX_RETRY_CONNECT) {
setTimeout(
() => {
if (this.canReconnect()) this.socket.socket.connect()
},
// slow down when potentially offline
(navigator.onLine ? 0 : BACK_OFF_RECONNECT_OFFLINE) +
// add jitter to spread reconnects
this.connectionAttempt *
(1 + Math.random()) *
CONNECTION_ERROR_RECONNECT_DELAY
)
} else {
if (!this.switchToWsFallbackIfPossible()) {
this.disconnect()
this.changeState({
...this.state,
error: 'unable-to-connect',
})
}
}
}
private onConnect() {
if (externalSocketHeartbeat) {
if (this.externalHeartbeatInterval) {
window.clearInterval(this.externalHeartbeatInterval)
}
if (this.socket.socket.transport?.name === 'websocket') {
// Do not enable external heartbeat on polling transports.
this.externalHeartbeatInterval = window.setInterval(
() => this.sendExternalHeartbeat(),
15_000
)
}
}
// Reset on success regardless of transport. We want to upgrade back to websocket on reconnect.
this.websocketFailureCount = 0
}
private onDisconnect() {
this.connectionAttempt = null
if (this.externalHeartbeatInterval) {
window.clearInterval(this.externalHeartbeatInterval)
}
this.externalHeartbeat.currentStart = 0
this.changeState({
...this.state,
readyState: WebSocket.CLOSED,
})
if (this.disconnectIfIdleSince(DISCONNECT_AFTER_MS)) return
if (this.state.error === 'rate-limited') {
this.tryReconnectWithBackoff(JOIN_PROJECT_RATE_LIMITED_DELAY)
} else {
this.startAutoReconnectCountdown(0)
}
}
private onForceDisconnect(
delay: number,
error: ConnectionError = 'maintenance'
) {
clearInterval(this.idleDisconnectInterval)
clearTimeout(this.reconnectCountdownInterval)
window.removeEventListener('online', this.onOnline)
window.setTimeout(() => this.disconnect(), 1000 * delay)
this.changeState({
...this.state,
forceDisconnected: true,
forcedDisconnectDelay: delay,
error,
})
}
private onJoinProjectResponse({
protocolVersion,
publicId,
}: {
protocolVersion: number
publicId: string
}) {
if (
this.protocolVersion !== -1 &&
this.protocolVersion !== protocolVersion
) {
this.onForceDisconnect(0, 'protocol-changed')
return
}
this.protocolVersion = protocolVersion
this.socket.publicId = publicId
this.connectionAttempt = null
this.changeState({
...this.state,
readyState: WebSocket.OPEN,
error: '',
reconnectAt: null,
})
}
private onReconnectGracefully() {
// Disconnect idle users a little earlier than the 24h limit.
if (this.disconnectIfIdleSince(DISCONNECT_AFTER_MS * 0.75)) return
if (this.gracefullyReconnectUntil) return
this.gracefullyReconnectUntil =
performance.now() + MAX_RECONNECT_GRACEFULLY_INTERVAL_MS
this.tryReconnectGracefully()
}
private canReconnect(): boolean {
if (this.state.readyState === WebSocket.OPEN) return false // no need to reconnect
if (this.state.forceDisconnected) return false // reconnecting blocked
return true
}
private isReconnectingSoon(ms: number): boolean {
if (!this.state.reconnectAt) return false
return this.state.reconnectAt - performance.now() <= ms
}
private hasReconnectedRecently(ms: number): boolean {
return performance.now() - this.state.lastConnectionAttempt < ms
}
private isUserInactiveSince(since: number): boolean {
return performance.now() - this.lastUserActivity > since
}
private disconnectIfIdleSince(threshold: number): boolean {
if (!this.isUserInactiveSince(threshold)) return false
const previouslyClosed = this.state.readyState === WebSocket.CLOSED
this.changeState({
...this.state,
readyState: WebSocket.CLOSED,
inactiveDisconnect: true,
})
if (!previouslyClosed) {
this.socket.disconnect()
}
return true
}
private disconnect() {
this.changeState({
...this.state,
readyState: WebSocket.CLOSED,
})
this.socket.disconnect()
}
private ensureIsConnected() {
if (this.state.readyState === WebSocket.OPEN) return
this.tryReconnectWithBackoff(
this.state.error === 'rate-limited'
? JOIN_PROJECT_RATE_LIMITED_DELAY
: USER_ACTIVITY_RECONNECT_DELAY
)
}
private startAutoReconnectCountdown(backoff: number) {
if (this.userIsLeavingPage) return
if (!this.canReconnect()) return
let countdown
if (this.isUserInactiveSince(TWO_MINUTES_IN_MS)) {
countdown = 60 + Math.floor(Math.random() * 2 * 60)
} else {
countdown = 3 + Math.floor(Math.random() * 7)
}
const ms = backoff + countdown * 1000
if (this.isReconnectingSoon(ms)) return
this.changeState({
...this.state,
reconnectAt: performance.now() + ms,
})
clearTimeout(this.reconnectCountdownInterval)
this.reconnectCountdownInterval = window.setTimeout(() => {
if (this.isReconnectingSoon(0)) {
this.tryReconnect()
}
}, ms)
}
private tryReconnect() {
this.gracefullyReconnectUntil = 0
this.changeState({
...this.state,
reconnectAt: null,
})
if (!this.canReconnect()) return
this.connectionAttempt = 0
this.changeState({
...this.state,
readyState: WebSocket.CONNECTING,
error: '',
inactiveDisconnect: false,
lastConnectionAttempt: performance.now(),
})
this.addReconnectListeners()
this.socket.socket.transports = ['xhr-polling']
if (this.websocketFailureCount < RETRY_WEBSOCKET) {
this.socket.socket.transports.unshift('websocket')
}
if (this.socket.socket.connecting || this.socket.socket.connected) {
// Ensure the old transport has been cleaned up.
// Socket.disconnect() does not accept a parameter. Go one level deeper.
this.socket.forceDisconnectWithoutEvent()
}
this.socket.socket.connect()
}
private addReconnectListeners() {
const handleFailure = () => {
removeSocketListeners()
this.startAutoReconnectCountdown(
// slow down when potentially offline
navigator.onLine ? 0 : BACK_OFF_RECONNECT_OFFLINE
)
}
const handleSuccess = () => {
removeSocketListeners()
}
const removeSocketListeners = () => {
this.socket.removeListener('error', handleFailure)
this.socket.removeListener('connect', handleSuccess)
}
this.socket.on('error', handleFailure)
this.socket.on('connect', handleSuccess)
}
private tryReconnectGracefully() {
if (
this.state.readyState === WebSocket.CLOSED ||
!this.gracefullyReconnectUntil
)
return
if (
this.gracefullyReconnectUntil < performance.now() ||
this.isUserInactiveSince(RECONNECT_GRACEFULLY_RETRY_INTERVAL_MS)
) {
this.disconnect()
this.tryReconnect()
} else {
setTimeout(() => {
this.tryReconnectGracefully()
}, RECONNECT_GRACEFULLY_RETRY_INTERVAL_MS)
}
}
private tryReconnectWithBackoff(backoff: number) {
if (this.hasReconnectedRecently(backoff)) {
this.startAutoReconnectCountdown(backoff)
} else {
this.tryReconnect()
}
}
private sendExternalHeartbeat() {
const t0 = performance.now()
this.socket.emit('debug.getHostname', () => {
if (this.externalHeartbeat.currentStart !== t0) {
return
}
const t1 = performance.now()
this.externalHeartbeat = {
currentStart: 0,
lastSuccess: t1,
lastLatency: t1 - t0,
}
})
this.externalHeartbeat.currentStart = t0
}
private sendPingResponse(
counter?: number,
timestamp?: number,
serverTransport?: string,
serverSessionId?: string
) {
const clientTransport = this.socket.socket.transport?.name
const clientSessionId = this.socket.socket.sessionid
this.socket.emit(
'clientPong',
counter,
timestamp,
serverTransport,
serverSessionId,
clientTransport,
clientSessionId
)
}
}

View File

@@ -0,0 +1,235 @@
/*
Migrated from services/web/frontend/js/ide/connection/EditorWatchdogManager.js
EditorWatchdogManager is used for end-to-end checks of edits.
The editor UI is backed by Ace and CodeMirrors, which in turn are connected
to ShareJs documents in the frontend.
Edits propagate from the editor to ShareJs and are send through socket.io
and real-time to document-updater.
In document-updater edits are integrated into the document history and
a confirmation/rejection is sent back to the frontend.
Along the way things can get lost.
We have certain safe-guards in place, but are still getting occasional
reports of lost edits.
EditorWatchdogManager is implementing the basis for end-to-end checks on
two levels:
- local/ShareJsDoc: edits that pass-by a ShareJs document shall get
acknowledged eventually.
- global: any edits made in the editor shall get acknowledged eventually,
independent for which ShareJs document (potentially none) sees it.
How does this work?
===================
The global check is using a global EditorWatchdogManager that is available
via the angular factory 'ide'.
Local/ShareJsDoc level checks will connect to the global instance.
Each EditorWatchdogManager keeps track of the oldest un-acknowledged edit.
When ever a ShareJs document receives an acknowledgement event, a local
EditorWatchdogManager will see it and also notify the global instance about
it.
The next edit cycle will clear the oldest un-acknowledged timestamp in case
a new ack has arrived, otherwise it will bark loud! via the timeout handler.
Scenarios
=========
- User opens the CodeMirror editor
- attach global check to new CM instance
- detach Ace from the local EditorWatchdogManager
- when the frontend attaches the CM instance to ShareJs, we also
attach it to the local EditorWatchdogManager
- the internal attach process writes the document content to the editor,
which in turn emits 'change' events. These event need to be excluded
from the watchdog. EditorWatchdogManager.ignoreEditsFor takes care
of that.
- User opens the Ace editor (again)
- (attach global check to the Ace editor, only one copy of Ace is around)
- detach local EditorWatchdogManager from CM
- likewise with CM, attach Ace to the local EditorWatchdogManager
- User makes an edit
- the editor will emit a 'change' event
- the global EditorWatchdogManager will process it first
- the local EditorWatchdogManager will process it next
- Document-updater confirms an edit
- the local EditorWatchdogManager will process it first, it passes it on to
- the global EditorWatchdogManager will process it next
Time
====
The delay between edits and acks is measured using a monotonic clock:
`performance.now()`.
It is agnostic to system clock changes in either direction and timezone
changes do not affect it as well.
Roughly speaking, it is initialized with `0` when the `window` context is
created, before our JS app boots.
As per canIUse.com and MDN `performance.now()` is available to all supported
Browsers, including IE11.
See also: https://caniuse.com/?search=performance.now
See also: https://developer.mozilla.org/en-US/docs/Web/API/Performance/now
*/
import {
ChangeDescription,
EditorFacade,
} from '../../source-editor/extensions/realtime'
import { debugConsole } from '@/utils/debugging'
// TIMEOUT specifies the timeout for edits into a single ShareJsDoc.
const TIMEOUT = 60 * 1000
// GLOBAL_TIMEOUT specifies the timeout for edits into any ShareJSDoc.
const GLOBAL_TIMEOUT = TIMEOUT
// REPORT_EVERY specifies how often we send events/report errors.
const REPORT_EVERY = 60 * 1000
const SCOPE_LOCAL = 'ShareJsDoc'
const SCOPE_GLOBAL = 'global'
type Scope = 'ShareJsDoc' | 'global'
type Meta = {
scope: Scope
delay: number
lastAck: number
lastUnackedEdit: number
}
type TimeoutHandler = (meta: Meta) => void
class Reporter {
private lastReport: number | null = null
private queue: Meta[] = []
// eslint-disable-next-line no-useless-constructor
constructor(private readonly onTimeoutHandler: TimeoutHandler) {}
private getMetaPreferLocal() {
for (const meta of this.queue) {
if (meta.scope === SCOPE_LOCAL) {
return meta
}
}
return this.queue.pop()
}
onTimeout(meta: Meta) {
// Collect all 'meta's for this update.
// global arrive before local ones, but we are eager to report local ones.
this.queue.push(meta)
setTimeout(() => {
// Another handler processed the 'meta' entry already
if (!this.queue.length) return
// There is always an item on the queue at this point,
// so getMetaPreferLocal will always return a Meta object
const maybeLocalMeta = this.getMetaPreferLocal() as Meta
// Discard other, newly arrived 'meta's
this.queue.length = 0
const now = Date.now()
// Do not flood the server with losing-edits events
const reportedRecently =
this.lastReport !== null && now - this.lastReport < REPORT_EVERY
if (!reportedRecently) {
this.lastReport = now
this.onTimeoutHandler(maybeLocalMeta)
}
})
}
}
export default class EditorWatchdogManager {
lastAck: number | null = null
reporter: Reporter
parent?: EditorWatchdogManager
scope: Scope
timeout: number
lastUnackedEdit: number | null
constructor({
parent,
onTimeoutHandler,
}: {
parent?: EditorWatchdogManager
onTimeoutHandler?: TimeoutHandler
}) {
this.scope = parent ? SCOPE_LOCAL : SCOPE_GLOBAL
this.timeout = parent ? TIMEOUT : GLOBAL_TIMEOUT
this.parent = parent
if (parent) {
this.reporter = parent.reporter
} else if (onTimeoutHandler) {
this.reporter = new Reporter(onTimeoutHandler)
} else {
throw new Error('No parent or onTimeoutHandler')
}
this.lastAck = null
this.lastUnackedEdit = null
}
onAck() {
this.lastAck = performance.now()
// bubble up to globalEditorWatchdogManager
if (this.parent) this.parent.onAck()
}
onEdit() {
// Use timestamps to track the high-water mark of unacked edits
const now = performance.now()
// Discard the last unacked edit if there are now newer acks
// TODO Handle cases where lastAck and/or lastUnackedEdit are null more transparently
// @ts-ignore
if (this.lastAck > this.lastUnackedEdit) {
this.lastUnackedEdit = null
}
// Start tracking for this keypress if we aren't already tracking an
// unacked edit
if (!this.lastUnackedEdit) {
this.lastUnackedEdit = now
}
// Report an error if the last tracked edit hasn't been cleared by an
// ack from the server after a long time
const delay = now - this.lastUnackedEdit
if (delay > this.timeout) {
const timeOrigin = Date.now() - now
const scope = this.scope
const lastAck = this.lastAck ? timeOrigin + this.lastAck : 0
const lastUnackedEdit = timeOrigin + this.lastUnackedEdit
const meta: Meta = { scope, delay, lastAck, lastUnackedEdit }
this.log('timedOut', meta)
this.reporter.onTimeout(meta)
}
}
attachToEditor(editor: EditorFacade) {
this.log('attach to editor')
const onChange = (
_editor: EditorFacade,
changeDescription: ChangeDescription
) => {
if (changeDescription.origin === 'remote') return
if (!(changeDescription.removed || changeDescription.inserted)) return
this.onEdit()
}
editor.on('change', onChange)
return () => {
this.log('detach from editor')
editor.off('change', onChange)
}
}
private log(...args: any[]) {
debugConsole.log(`[EditorWatchdogManager] ${this.scope}:`, ...args)
}
}

View File

@@ -0,0 +1,14 @@
import { Project } from '../../../../../types/project'
import { PermissionsLevel } from '@/features/ide-react/types/permissions'
export type JoinProjectPayloadProject = Pick<
Project,
Exclude<keyof Project, ['rootDocId', 'publicAccessLevel']>
> & { rootDoc_id?: string; publicAccesLevel?: string }
export type JoinProjectPayload = {
permissionsLevel: PermissionsLevel
project: JoinProjectPayloadProject
protocolVersion: number
publicId: string
}

View File

@@ -0,0 +1,35 @@
export type ConnectionError =
| 'io-not-loaded'
| 'maintenance'
| 'not-logged-in'
| 'out-of-sync'
| 'project-deleted'
| 'protocol-changed'
| 'rate-limited'
| 'unable-to-connect'
| 'unable-to-join'
export type ConnectionState = {
readyState: WebSocket['CONNECTING'] | WebSocket['OPEN'] | WebSocket['CLOSED']
forceDisconnected: boolean
inactiveDisconnect: boolean
reconnectAt: number | null
forcedDisconnectDelay: number
lastConnectionAttempt: number
error: '' | ConnectionError
}
export type ExternalHeartbeat = {
currentStart: number
lastSuccess: number
lastLatency: number
}
export type SocketDebuggingInfo = {
client_id?: string
publicId?: string
transport?: string
lastUserActivity: number
connectionState: ConnectionState
externalHeartbeat: ExternalHeartbeat
}

View File

@@ -0,0 +1,46 @@
export type Socket = {
publicId: string
on(event: string, callback: (...data: any[]) => void): void
removeListener(event: string, callback: (...data: any[]) => void): void
emit(
event: string,
arg0: any,
callback?: (error: Error, ...data: any[]) => void
): void
emit(
event: string,
arg0: any,
arg1: any,
callback?: (error: Error, ...data: any[]) => void
): void
emit(
event: string,
arg0: any,
arg1: any,
arg2: any,
callback?: (error: Error, ...data: any[]) => void
): void
emit(
event: string,
arg0: any,
arg1: any,
arg2: any,
arg3: any,
arg4: any,
arg5: any,
callback?: (error: Error, ...data: any[]) => void
): void
socket: {
connected: boolean
connecting: boolean
connect(): void
disconnect(): void
sessionid: string
transport?: {
name: string
}
transports: string[]
}
disconnect(): void
forceDisconnectWithoutEvent(): void
}

View File

@@ -0,0 +1,6 @@
export function secondsUntil(timestamp: number | null) {
if (!timestamp) return 0
const seconds = Math.ceil((timestamp - performance.now()) / 1000)
if (seconds > 0) return seconds
return 0
}