Merge pull request #1169 from bbc/feat/mos-openmedia-hot-standby

feat(MOS): support OpenMedia's hot standby

packages/mos-gateway/src/$schemas/devices.json

@@ -61,13 +61,20 @@
 							"ui:title": "(Optional) MOS Query Port",
 							"ui:description": "Connect to an alternate port for 'query' port MOS messages",
 							"default": 10542
-						}	
+						}
 					},
-					"required": ["lower", "upper", "query"],
+					"required": [
+						"lower",
+						"upper",
+						"query"
+					],
 					"additionalProperties": false
 				}
 			},
-			"required": ["id", "host"],
+			"required": [
+				"id",
+				"host"
+			],
 			"additionalProperties": false
 		},
 		"secondary": {
@@ -105,6 +112,12 @@
 					"ui:description": "How often to ping NRCS to determine connection status",
 					"default": 30000
 				},
+				"openMediaHotStandby": {
+					"type": "boolean",
+					"ui:title": "Secondary: OpenMedia Hot Standby",
+					"ui:description": "Is the secondary connection a OpenMedia hot standby for the primary",
+					"default": false
+				},
 				"ports": {
 					"type": "object",
 					"ui:title": "Ports",
@@ -126,16 +139,25 @@
 							"ui:title": "(Optional) MOS Query Port",
 							"ui:description": "Connect to an alternate port for 'query' port MOS messages",
 							"default": 10542
-						}	
+						}
 					},
-					"required": ["lower", "upper", "query"],
+					"required": [
+						"lower",
+						"upper",
+						"query"
+					],
 					"additionalProperties": false
-				}				
+				}
 			},
-			"required": ["id", "host"],
+			"required": [
+				"id",
+				"host"
+			],
 			"additionalProperties": false
 		}
 	},
-	"required": ["primary"],
+	"required": [
+		"primary"
+	],
 	"additionalProperties": false
-}
+}

packages/mos-gateway/src/CoreMosDeviceHandler.ts

@@ -62,6 +62,10 @@ interface IStoryItemChange {
 	itemDiff: PartialDeep<IMOSItem>
 }
 
+export interface CoreMosDeviceHandlerOptions {
+	openMediaHotStandby?: boolean
+}
+
 /**
  * Represents a connection between a mos-device and Core
  */
@@ -76,13 +80,20 @@ export class CoreMosDeviceHandler {
 	private _pendingStoryItemChanges: Array<IStoryItemChange> = []
 	private _pendingChangeTimeout: number = 60 * 1000
 	private mosTypes: MosTypes
+	private _options: CoreMosDeviceHandlerOptions
 
 	private _messageQueue: Queue
 
-	constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler) {
+	constructor(
+		parent: CoreHandler,
+		mosDevice: IMOSDevice,
+		mosHandler: MosHandler,
+		options: CoreMosDeviceHandlerOptions
+	) {
 		this._coreParentHandler = parent
 		this._mosDevice = mosDevice
 		this._mosHandler = mosHandler
+		this._options = options
 
 		this._messageQueue = new Queue()
 
@@ -139,25 +150,47 @@ export class CoreMosDeviceHandler {
 		let statusCode: StatusCode
 		const messages: Array<string> = []
 
-		if (connectionStatus.PrimaryConnected) {
-			if (connectionStatus.SecondaryConnected || !this._mosDevice.idSecondary) {
+		if (this._options.openMediaHotStandby) {
+			// OpenMedia treats secondary server as hot-standby
+			// And thus is not considered as a warning if it's not connected
+			if (connectionStatus.PrimaryConnected) {
 				statusCode = StatusCode.GOOD
 			} else {
-				statusCode = StatusCode.WARNING_MINOR
+				// Primary not connected is only bad if there is no secondary:
+				if (connectionStatus.SecondaryConnected) {
+					statusCode = StatusCode.GOOD
+					messages.push(connectionStatus.SecondaryStatus || 'Running NRCS on hot standby')
+				} else {
+					statusCode = StatusCode.BAD
+					// Send messages for both connections
+					messages.push(connectionStatus.PrimaryStatus || 'Primary and hot standby are not connected')
+					messages.push(connectionStatus.SecondaryStatus || 'Primary and hot standby are not connected')
+				}
 			}
 		} else {
-			if (connectionStatus.SecondaryConnected) {
-				statusCode = StatusCode.WARNING_MAJOR
+			if (connectionStatus.PrimaryConnected) {
+				// ENPS expect both Primary and Secondary to be connected if both of them are configured
+				if (connectionStatus.SecondaryConnected || !this._mosDevice.idSecondary) {
+					statusCode = StatusCode.GOOD
+				} else {
+					statusCode = StatusCode.WARNING_MINOR
+				}
 			} else {
-				statusCode = StatusCode.BAD
+				if (connectionStatus.SecondaryConnected) {
+					// Primary not connected should give a warning if Secondary is used.
+					statusCode = StatusCode.WARNING_MAJOR
+				} else {
+					// If neither Primary nor Secondary is connected, it's a bad state.
+					statusCode = StatusCode.BAD
+				}
 			}
-		}
 
-		if (!connectionStatus.PrimaryConnected) {
-			messages.push(connectionStatus.PrimaryStatus || 'Primary not connected')
-		}
-		if (this._mosDevice.idSecondary && !connectionStatus.SecondaryConnected) {
-			messages.push(connectionStatus.SecondaryStatus || 'Fallback not connected')
+			if (!connectionStatus.PrimaryConnected) {
+				messages.push(connectionStatus.PrimaryStatus || 'Primary not connected')
+			}
+			if (this._mosDevice.idSecondary && !connectionStatus.SecondaryConnected) {
+				messages.push(connectionStatus.SecondaryStatus || 'Fallback not connected')
+			}
 		}
 
 		this.core

packages/mos-gateway/src/coreHandler.ts

@@ -18,7 +18,7 @@ import { MosHandler } from './mosHandler'
 import { DeviceConfig } from './connector'
 import { MOS_DEVICE_CONFIG_MANIFEST } from './configManifest'
 import { getVersions } from './versions'
-import { CoreMosDeviceHandler } from './CoreMosDeviceHandler'
+import { CoreMosDeviceHandler, CoreMosDeviceHandlerOptions } from './CoreMosDeviceHandler'
 import { PeripheralDeviceCommandId } from '@sofie-automation/shared-lib/dist/core/model/Ids'
 
 export interface CoreConfig {
@@ -142,9 +142,13 @@ export class CoreHandler {
 
 		return options
 	}
-	async registerMosDevice(mosDevice: IMOSDevice, mosHandler: MosHandler): Promise<CoreMosDeviceHandler> {
+	async registerMosDevice(
+		mosDevice: IMOSDevice,
+		mosHandler: MosHandler,
+		deviceOptions: CoreMosDeviceHandlerOptions
+	): Promise<CoreMosDeviceHandler> {
 		this.logger.info('registerMosDevice -------------')
-		const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler)
+		const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, deviceOptions)
 
 		this._coreMosHandlers.push(coreMos)
 		return coreMos.init().then(() => {

packages/mos-gateway/src/generated/devices.ts

@@ -24,6 +24,7 @@ export interface MosDeviceConfig {
 		dontUseQueryPort?: boolean
 		timeout?: number
 		heartbeatInterval?: number
+		openMediaHotStandby?: boolean
 		ports?: {
 			lower: number
 			upper: number

packages/mos-gateway/src/mosHandler.ts

@@ -59,13 +59,15 @@ export class MosHandler {
 	private _logger: Winston.Logger
 	private _disposed = false
 	private _settings?: MosGatewayConfig
+	private _openMediaHotStandby: Record<string, boolean>
 	private _coreHandler: CoreHandler | undefined
 	private _observers: Array<Observer<any>> = []
 	private _triggerupdateDevicesTimeout: any = null
 	private mosTypes: MosTypes
 
 	constructor(logger: Winston.Logger) {
 		this._logger = logger
+		this._openMediaHotStandby = {}
 		this.mosTypes = getMosTypes(this.strict) // temporary, another will be set upon init()
 	}
 	async init(config: MosConfig, coreHandler: CoreHandler): Promise<void> {
@@ -101,7 +103,7 @@ export class MosHandler {
 
 		this.mosTypes = getMosTypes(this.strict)
 
-		await this._initMosConnection()
+		await this._updateDevices()
 
 		if (!this._coreHandler) throw Error('_coreHandler is undefined!')
 		this._coreHandler.onConnected(() => {
@@ -110,8 +112,6 @@ export class MosHandler {
 			this.sendStatusOfAllMosDevices()
 		})
 		this.setupObservers()
-
-		return this._updateDevices()
 	}
 	async dispose(): Promise<void> {
 		this._disposed = true
@@ -243,7 +243,11 @@ export class MosHandler {
 
 				if (!this._coreHandler) throw Error('_coreHandler is undefined!')
 
-				const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this)
+				const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this, {
+					openMediaHotStandby: mosDevice.idSecondary
+						? this._openMediaHotStandby[mosDevice.idSecondary]
+						: false,
+				})
 				// this._logger.info('mosDevice registered -------------')
 				// Setup message flow between the devices:
 
@@ -420,6 +424,8 @@ export class MosHandler {
 			for (const [deviceId, device] of Object.entries<{ options: MosDeviceConfig }>(devices)) {
 				if (device) {
 					if (device.options.secondary) {
+						this._openMediaHotStandby[device.options.secondary.id] =
+							device.options.secondary?.openMediaHotStandby || false
 						// If the host isn't set, don't use secondary:
 						if (!device.options.secondary.host || !device.options.secondary.id)
 							delete device.options.secondary
@@ -482,6 +488,11 @@ export class MosHandler {
 		deviceOptions.primary.heartbeatInterval =
 			deviceOptions.primary.heartbeatInterval || DEFAULT_MOS_HEARTBEAT_INTERVAL
 
+		if (deviceOptions.secondary?.id && this._openMediaHotStandby[deviceOptions.secondary.id]) {
+			//@ts-expect-error  this is not yet added to the official mos-connection
+			deviceOptions.secondary.openMediaHotStandby = true
+		}
+
 		const mosDevice: MosDevice = await this.mos.connect(deviceOptions)
 		this._ownMosDevices[deviceId] = mosDevice