chore: add unit test for Hot Standby feature

packages/connector/src/MosDevice.ts

@@ -231,6 +231,19 @@ export class MosDevice implements IMOSDevice {
 		return this._secondaryConnection ? this._secondaryConnection.id : null
 	}
 
+	/** @deprecated This is for unit tests only */
+	get connections(): {
+		primary: NCSServerConnection | null
+		secondary: NCSServerConnection | null
+		current: NCSServerConnection | null
+	} {
+		return {
+			primary: this._primaryConnection,
+			secondary: this._secondaryConnection,
+			current: this._currentConnection,
+		}
+	}
+
 	connect(): void {
 		if (this._primaryConnection) this._primaryConnection.connect()
 		if (this._secondaryConnection) this._secondaryConnection.connect()

packages/connector/src/__mocks__/socket.ts

@@ -51,6 +51,7 @@ export class SocketMock extends EventEmitter implements Socket {
 
 	private _responses: Array<ReplyTypes> = []
 	private _autoReplyToHeartBeat = true
+	public mockConnectCount = 0
 
 	constructor() {
 		super()
@@ -102,6 +103,7 @@ export class SocketMock extends EventEmitter implements Socket {
 	}
 	// @ts-expect-error mock
 	connect(port: number, host: string): this {
+		this.mockConnectCount++
 		this.connectedPort = port
 		this.connectedHost = host
 
@@ -197,6 +199,10 @@ export class SocketMock extends EventEmitter implements Socket {
 
 		this.emit('connect')
 	}
+	mockEmitClose(): void {
+		this.emit('close')
+	}
+
 	mockSentMessage0(data: unknown, encoding: string): void {
 		if (this._autoReplyToHeartBeat) {
 			const str: string = typeof data === 'string' ? data : this.decode(data as any)

packages/connector/src/__tests__/MosConnection.spec.ts

@@ -532,6 +532,156 @@ describe('MosDevice: General', () => {
 		expect(onError).toHaveBeenCalledTimes(0)
 		expect(onWarning).toHaveBeenCalledTimes(0)
 
+		await mos.dispose()
+	})
+	test('Hot standby', async () => {
+		const mos = new MosConnection({
+			mosID: 'jestMOS',
+			acceptsConnections: true,
+			profiles: {
+				'0': true,
+				'1': true,
+			},
+		})
+		const onError = jest.fn((e) => console.log(e))
+		const onWarning = jest.fn((e) => console.log(e))
+		mos.on('error', onError)
+		mos.on('warning', onWarning)
+
+		expect(mos.acceptsConnections).toBe(true)
+		await initMosConnection(mos)
+		expect(mos.isListening).toBe(true)
+
+		const mosDevice = await mos.connect({
+			primary: {
+				id: 'primary',
+				host: '192.168.0.1',
+				timeout: 200,
+			},
+			secondary: {
+				id: 'secondary',
+				host: '192.168.0.2',
+				timeout: 200,
+				isHotStandby: true,
+			},
+		})
+
+		expect(mosDevice).toBeTruthy()
+		expect(mosDevice.idPrimary).toEqual('jestMOS_primary')
+
+		expect(mosDevice.connections.primary).toBeTruthy()
+		expect(mosDevice.connections.secondary).toBeTruthy()
+		mosDevice.connections.primary?.setAutoReconnectInterval(300)
+		mosDevice.connections.secondary?.setAutoReconnectInterval(300)
+
+		const onConnectionChange = jest.fn()
+		mosDevice.onConnectionChange((connectionStatus: IMOSConnectionStatus) => {
+			onConnectionChange(connectionStatus)
+		})
+
+		expect(SocketMock.instances).toHaveLength(7)
+		expect(SocketMock.instances[1].connectedHost).toEqual('192.168.0.1')
+		expect(SocketMock.instances[1].connectedPort).toEqual(10540)
+		expect(SocketMock.instances[2].connectedHost).toEqual('192.168.0.1')
+		expect(SocketMock.instances[2].connectedPort).toEqual(10541)
+		expect(SocketMock.instances[3].connectedHost).toEqual('192.168.0.1')
+		expect(SocketMock.instances[3].connectedPort).toEqual(10542)
+
+		// TODO: Perhaps the hot-standby should not be connected at all at this point?
+		expect(SocketMock.instances[4].connectedHost).toEqual('192.168.0.2')
+		expect(SocketMock.instances[4].connectedPort).toEqual(10540)
+		expect(SocketMock.instances[5].connectedHost).toEqual('192.168.0.2')
+		expect(SocketMock.instances[5].connectedPort).toEqual(10541)
+		expect(SocketMock.instances[6].connectedHost).toEqual('192.168.0.2')
+		expect(SocketMock.instances[6].connectedPort).toEqual(10542)
+
+		// Simulate primary connected:
+		for (const i of SocketMock.instances) {
+			if (i.connectedHost === '192.168.0.1') i.mockEmitConnected()
+		}
+		// Wait for the primary to be initially connected:
+		await waitFor(() => mosDevice.getConnectionStatus().PrimaryConnected, 1000)
+
+		// Check that the connection status is as we expect:
+		expect(mosDevice.getConnectionStatus()).toMatchObject({
+			PrimaryConnected: true,
+			PrimaryStatus: 'Primary: Connected',
+			SecondaryConnected: true, // Is a hot standby, so we pretend that it is connected
+			SecondaryStatus: 'Secondary: Is hot Standby',
+		})
+		expect(onConnectionChange).toHaveBeenCalled()
+		expect(onConnectionChange).toHaveBeenLastCalledWith({
+			PrimaryConnected: true,
+			PrimaryStatus: 'Primary: Connected',
+			SecondaryConnected: true, // Is a hot standby, so we pretend that it is connected
+			SecondaryStatus: 'Secondary: Is hot Standby',
+		})
+		onConnectionChange.mockClear()
+
+		// Simulate primary disconnect, secondary hot standby takes over:
+		for (const i of SocketMock.instances) {
+			i.mockConnectCount = 0
+			if (i.connectedHost === '192.168.0.1') i.mockEmitClose()
+			if (i.connectedHost === '192.168.0.2') i.mockEmitConnected()
+		}
+
+		// Wait for the secondary to be connected:
+		await waitFor(() => mosDevice.getConnectionStatus().SecondaryConnected, 1000)
+
+		// Check that the connection status is as we expect:
+		expect(mosDevice.getConnectionStatus()).toMatchObject({
+			PrimaryConnected: false,
+			PrimaryStatus: expect.stringContaining('Primary'),
+			SecondaryConnected: true,
+			SecondaryStatus: 'Secondary: Connected',
+		})
+		expect(onConnectionChange).toHaveBeenCalled()
+		expect(onConnectionChange).toHaveBeenLastCalledWith({
+			PrimaryConnected: false,
+			PrimaryStatus: expect.stringContaining('Primary'),
+			SecondaryConnected: true,
+			SecondaryStatus: 'Secondary: Connected',
+		})
+		onConnectionChange.mockClear()
+
+		// Simulate that the primary comes back online:
+		for (const i of SocketMock.instances) {
+			if (i.connectedHost === '192.168.0.1') {
+				expect(i.mockConnectCount).toBeGreaterThanOrEqual(1) // should have tried to reconnect
+				i.mockEmitConnected()
+			}
+
+			if (i.connectedHost === '192.168.0.2') i.mockEmitClose()
+		}
+
+		// Wait for the primary to be connected:
+		await waitFor(() => mosDevice.getConnectionStatus().PrimaryConnected, 1000)
+
+		// Check that the connection status is as we expect:
+		expect(mosDevice.getConnectionStatus()).toMatchObject({
+			PrimaryConnected: true,
+			PrimaryStatus: 'Primary: Connected',
+			SecondaryConnected: true, // Is a hot standby, so we pretend that it is connected
+			SecondaryStatus: 'Secondary: Is hot Standby',
+		})
+		expect(onConnectionChange).toHaveBeenCalled()
+		expect(onConnectionChange).toHaveBeenLastCalledWith({
+			PrimaryConnected: true,
+			PrimaryStatus: 'Primary: Connected',
+			SecondaryConnected: true, // Is a hot standby, so we pretend that it is connected
+			SecondaryStatus: 'Secondary: Is hot Standby',
+		})
+
 		await mos.dispose()
 	})
 })
+async function waitFor(fcn: () => boolean, timeout: number): Promise<void> {
+	const startTime = Date.now()
+
+	while (Date.now() - startTime < timeout) {
+		await delay(10)
+
+		if (fcn()) return
+	}
+	throw new Error('Timeout in waitFor')
+}

packages/connector/src/connection/NCSServerConnection.ts

@@ -120,6 +120,12 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 		}
 	}
 
+	setAutoReconnectInterval(interval: number): void {
+		for (const i in this._clients) {
+			this._clients[i].client.autoReconnectInterval = interval
+		}
+	}
+
 	connect(): void {
 		for (const i in this._clients) {
 			// Connect client