diff --git a/client/.dockerignore b/client/.dockerignore index baca19af..44d0fd3a 100644 --- a/client/.dockerignore +++ b/client/.dockerignore @@ -13,3 +13,4 @@ configure/*.local *~ .*.swp +.ci diff --git a/client/Dockerfile b/client/Dockerfile index 714f2d8d..3aa7c419 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -50,8 +50,9 @@ RUN mv docker/RELEASE.local configure/RELEASE.local ENV EPICS_ROOT=/epics ENV EPICS_BASE=${EPICS_ROOT}/base RUN make -WORKDIR /recsync/iocBoot/iocdemo FROM recsync-base AS ioc-runner -CMD /recsync/bin/${EPICS_HOST_ARCH}/demo st.cmd +WORKDIR /recsync/bin/${EPICS_HOST_ARCH} + +CMD ./demo /recsync/iocBoot/iocdemo/st.cmd diff --git a/client/demoApp/Db/Makefile b/client/demoApp/Db/Makefile index 309d4854..60db2767 100644 --- a/client/demoApp/Db/Makefile +++ b/client/demoApp/Db/Makefile @@ -11,7 +11,8 @@ include $(TOP)/configure/CONFIG # Create and install (or just install) into /db # databases, templates, substitutions like this DB += somerecords.db - +DB += archive.db +DB += archive_bugtest.db #---------------------------------------------------- # If .db template is not named *.template add # _template = diff --git a/client/demoApp/Db/archive_bugtest.db b/client/demoApp/Db/archive_bugtest.db new file mode 100644 index 00000000..833e0bc6 --- /dev/null +++ b/client/demoApp/Db/archive_bugtest.db @@ -0,0 +1,10 @@ + +record(ai, "$(P)ai:archive") { + info("test", "testing") +} + +record(longout, "$(P)lo:archive") { + info("test", "testing") + info("hello", "world") + info("archive", "default") +} diff --git a/client/example-compose.yml b/client/ioc-compose.yml similarity index 65% rename from client/example-compose.yml rename to client/ioc-compose.yml index 7b7ca6e6..881c4f56 100644 --- a/client/example-compose.yml +++ b/client/ioc-compose.yml @@ -3,11 +3,11 @@ services: ioc1: build: ../client environment: - - IOCSH_NAME=IOC1-2 + - IOCSH_NAME=IOC1 tty: true networks: - - net-recc-1 + - net-1-recc-1 networks: - net-recc-1: + net-1-recc-1: driver: bridge diff --git a/client/iocBoot/iocdemo/st.cmd b/client/iocBoot/iocdemo/st.cmd index 071829fa..fe81b085 100755 --- a/client/iocBoot/iocdemo/st.cmd +++ b/client/iocBoot/iocdemo/st.cmd @@ -26,5 +26,6 @@ addReccasterEnvVars("BUILDING") ## Load record instances dbLoadRecords("../../db/reccaster.db", "P=$(IOCSH_NAME):") dbLoadRecords("../../db/somerecords.db","P=$(IOCSH_NAME):") +dbLoadRecords("../../db/archive.db", "P=$(IOCSH_NAME):") iocInit() diff --git a/client/iocBoot/iocdemo/st_bugtest.cmd b/client/iocBoot/iocdemo/st_bugtest.cmd new file mode 100644 index 00000000..c1e16157 --- /dev/null +++ b/client/iocBoot/iocdemo/st_bugtest.cmd @@ -0,0 +1,31 @@ +#!../../bin/linux-x86_64-debug/demo + +## You may have to change demo to something else +## everywhere it appears in this file + +< envPaths + +## Register all support components +dbLoadDatabase("../../dbd/demo.dbd",0,0) +demo_registerRecordDeviceDriver(pdbbase) + +var(reccastTimeout, 5.0) +var(reccastMaxHoldoff, 5.0) + +epicsEnvSet("IOCNAME", "$(IOCSH_NAME)") +epicsEnvSet("ENGINEER", "myself") +epicsEnvSet("LOCATION", "myplace") + +epicsEnvSet("CONTACT", "mycontact") +epicsEnvSet("BUILDING", "mybuilding") +epicsEnvSet("SECTOR", "mysector") + +addReccasterEnvVars("CONTACT", "SECTOR") +addReccasterEnvVars("BUILDING") + +## Load record instances +dbLoadRecords("../../db/reccaster.db", "P=$(IOCSH_NAME):") +dbLoadRecords("../../db/somerecords.db","P=$(IOCSH_NAME):") +dbLoadRecords("../../db/archive_bugtest.db", "P=$(IOCSH_NAME):") + +iocInit() diff --git a/server/compose.yml b/server/compose.yml new file mode 100644 index 00000000..af5acc34 --- /dev/null +++ b/server/compose.yml @@ -0,0 +1,30 @@ +include: + - docker/cf-compose.yml +services: + recc: + build: . + depends_on: + cf: + condition: service_healthy + restart: true + healthcheck: + test: netstat | grep cf + interval: 10s + timeout: 30s + retries: 3 + volumes: + - type: bind + source: docker/config/cf1.conf + target: /home/recceiver/cf.conf + read_only: true + - type: bind + source: docker/config/cf.conf + target: /home/recceiver/channelfinderapi.conf + read_only: true + networks: + - net-1-recc-1 + - net-2-cf + +networks: + net-1-recc-1: + driver: bridge diff --git a/server/docker/cf-compose.yml b/server/docker/cf-compose.yml new file mode 100644 index 00000000..c14ac37f --- /dev/null +++ b/server/docker/cf-compose.yml @@ -0,0 +1,49 @@ +name: cf-compose +services: + cf: + image: ghcr.io/channelfinder/channelfinderservice:master + hostname: cf + networks: + - net-2-cf + ports: + - "8080:8080" + depends_on: + elasticsearch: + condition: service_healthy + restart: true + environment: + ELASTICSEARCH_NETWORK_HOST: elasticsearch-cf + ELASTICSEARCH_QUERY_SIZE: 10000 + demo_auth.enabled: true + demo_auth.users: admin + demo_auth.pwds: password + demo_auth.roles: ADMIN + EPICS_PVAS_INTF_ADDR_LIST: "0.0.0.0" + aa.enabled: false + logging.level.org.springframework.web: INFO + healthcheck: + test: curl -s -f http://cf:8080/ChannelFinder + interval: 10s + timeout: 60s + retries: 5 + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4 + hostname: elasticsearch-cf + networks: + - net-2-cf + environment: + cluster.name: channelfinder + discovery.type: single-node + bootstrap.memory_lock: "true" + xpack.security.enabled: "false" + logger.level: WARN + healthcheck: + test: curl -s -f http://localhost:9200/_cluster/health + interval: 10s + timeout: 60s + retries: 5 + +networks: + net-2-cf: + driver: bridge diff --git a/server/docker/config/cf.conf b/server/docker/config/cf.conf new file mode 100644 index 00000000..b7c239e4 --- /dev/null +++ b/server/docker/config/cf.conf @@ -0,0 +1,5 @@ + +[DEFAULT] +BaseURL = http://cf:8080/ChannelFinder +username = admin +password = password diff --git a/server/docker/config/cf1.conf b/server/docker/config/cf1.conf index c62a412f..b1ac4482 100644 --- a/server/docker/config/cf1.conf +++ b/server/docker/config/cf1.conf @@ -29,7 +29,5 @@ recordDesc = on # Mark all channels as 'Inactive' when processor is started (default: True) cleanOnStart = True -[DEFAULT] -BaseURL = http://cf:8080/ChannelFinder +# Name to used as owner username = admin -password = password diff --git a/server/docker/config/cf2.conf b/server/docker/config/cf2.conf index 635df032..18664199 100644 --- a/server/docker/config/cf2.conf +++ b/server/docker/config/cf2.conf @@ -29,7 +29,5 @@ recordDesc = on # Mark all channels as 'Inactive' when processor is started (default: True) cleanOnStart = True -[DEFAULT] -BaseURL = http://cf:8080/ChannelFinder +# Name to used as owner username = admin -password = password diff --git a/server/docker/test-multi-recc.yml b/server/docker/test-multi-recc.yml new file mode 100644 index 00000000..f66fa173 --- /dev/null +++ b/server/docker/test-multi-recc.yml @@ -0,0 +1,74 @@ +name: test-multi-recc +include: + - cf-compose.yml +services: + recc1: + extends: + file: ../compose.yml + service: recc + depends_on: + cf: + condition: service_healthy + restart: true + hostname: recc1 + networks: + - net-1-recc-1 + - net-2-cf + ioc1-1: + extends: + file: ../../client/ioc-compose.yml + service: ioc1 + depends_on: + recc1: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC1-1 + hostname: ioc1-1 + networks: + - net-1-recc-1 + ioc1-2: + extends: + ioc1-1 + depends_on: + recc1: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC1-2 + hostname: ioc1-1 + recc2: + extends: + recc1 + hostname: recc2 + networks: + - net-1-recc-2 + - net-2-cf + ioc2-1: + extends: + ioc1-1 + depends_on: + recc2: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC2-1 + hostname: ioc2-1 + networks: + - net-1-recc-2 + ioc2-2: + extends: + ioc2-1 + depends_on: + recc2: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC2-2 + hostname: ioc2-2 + +networks: + net-1-recc-1: + driver: bridge + net-1-recc-2: + driver: bridge diff --git a/server/docker/test-remove-infotag.yml b/server/docker/test-remove-infotag.yml new file mode 100644 index 00000000..ae4a43aa --- /dev/null +++ b/server/docker/test-remove-infotag.yml @@ -0,0 +1,34 @@ +name: test-removeinfotag +include: + - cf-compose.yml +services: + recc1: + extends: + file: ../compose.yml + service: recc + depends_on: + cf: + condition: service_healthy + restart: true + hostname: recc1 + networks: + - net-1-recc-1 + - net-2-cf + ioc1-1: + extends: + file: ../../client/ioc-compose.yml + service: ioc1 + environment: + - IOCSH_NAME=IOC1-1 + depends_on: + recc1: + condition: service_healthy + restart: true + hostname: ioc1-1 + command: bash + networks: + - net-1-recc-1 + +networks: + net-1-recc-1: + driver: bridge diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ce23e6f2..115a4e7c 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -124,6 +124,7 @@ def _startServiceWithLock(self): self.client.set(property={"name": cf_property, "owner": owner}) self.record_property_names_list = set(record_property_names_list) + self.managed_properties = required_properties.union(record_property_names_list) _log.debug("record_property_names_list = {}".format(self.record_property_names_list)) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") @@ -192,6 +193,7 @@ def _commitWithThread(self, transaction): ) _log.info("CF_COMMIT: {transaction}".format(transaction=transaction)) + _log.debug("CF_COMMIT: transaction: {s}".format(s=repr(transaction))) """ a dictionary with a list of records with their associated property info pvInfo @@ -212,6 +214,7 @@ def _commitWithThread(self, transaction): """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) + _log.debug("transaction: {s}".format(s=repr(transaction))) recordInfo = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): @@ -231,12 +234,9 @@ def _commitWithThread(self, transaction): if recinfo_wl: recordInfo[record_id]["infoProperties"] = list() for infotag in recinfo_wl: - property = { - "name": infotag, - "owner": owner, - "value": record_infos_to_add[infotag], - } - recordInfo[record_id]["infoProperties"].append(property) + recordInfo[record_id]["infoProperties"].append( + create_property(owner, infotag, record_infos_to_add[infotag]) + ) for record_id, alias in transaction.aliases.items(): if record_id not in recordInfo: @@ -251,14 +251,11 @@ def _commitWithThread(self, transaction): for record_id in recordInfo: for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: - property = { - "name": cf_prop_name, - "owner": owner, - "value": transaction.client_infos.get(epics_env_var_name), - } if "infoProperties" not in recordInfo[record_id]: recordInfo[record_id]["infoProperties"] = list() - recordInfo[record_id]["infoProperties"].append(property) + recordInfo[record_id]["infoProperties"].append( + create_property(owner, cf_prop_name, transaction.client_infos.get(epics_env_var_name)) + ) else: _log.debug( "EPICS environment var %s listed in environment_vars setting list not found in this IOC: %s", @@ -279,8 +276,6 @@ def _commitWithThread(self, transaction): ) continue recordInfoByName[info["pvName"]] = info - _log.debug("Add record: {record_id}: {info}".format(record_id=record_id, info=info)) - _log.debug("Add record: {record_id}: {info}".format(record_id=record_id, info=info)) if transaction.initial: """Add IOC to source list """ @@ -381,7 +376,7 @@ def clean_channels(self, owner, channels): 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) ) self.client.update( - property={"name": "pvStatus", "owner": owner, "value": "Inactive"}, + property=create_inactive_property(owner), channelNames=new_channels, ) @@ -401,6 +396,46 @@ def dict_to_file(dict, iocs, conf): json.dump(list, f) +def create_channel(name: str, owner: str, properties: list[dict[str, str]]): + return { + "name": name, + "owner": owner, + "properties": properties, + } + + +def create_property(owner: str, name: str, value: str): + return { + "name": name, + "owner": owner, + "value": value, + } + + +def create_recordType_property(owner: str, recordType: str): + return create_property(owner, "recordType", recordType) + + +def create_alias_property(owner: str, alias: str): + return create_property(owner, "alias", alias) + + +def create_pvStatus_property(owner: str, pvStatus: str): + return create_property(owner, "pvStatus", pvStatus) + + +def create_active_property(owner: str): + return create_pvStatus_property(owner, "Active") + + +def create_inactive_property(owner: str): + return create_pvStatus_property(owner, "Inactive") + + +def create_time_property(owner: str, time: str): + return create_property(owner, "time", time) + + def __updateCF__( processor, recordInfoByName, @@ -413,7 +448,11 @@ def __updateCF__( iocTime, ): _log.info("CF Update IOC: {iocid}".format(iocid=iocid)) - + _log.debug( + "CF Update IOC: {iocid} recordInfoByName {recordInfoByName}".format( + iocid=iocid, recordInfoByName=recordInfoByName + ) + ) # Consider making this function a class methed then 'processor' simply becomes 'self' client = processor.client channels_dict = processor.channel_dict @@ -449,22 +488,23 @@ def __updateCF__( if ( len(new_channels) == 0 or cf_channel["name"] in records_to_delete ): # case: empty commit/del, remove all reference to ioc + _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel["name"])) if cf_channel["name"] in channels_dict: cf_channel["owner"] = iocs[channels_dict[cf_channel["name"]][-1]]["owner"] cf_channel["properties"] = __merge_property_lists( - ch_create_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), - cf_channel["properties"], + create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), + cf_channel, + processor.managed_properties, ) if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( - { - "name": "recordType", - "owner": owner, - "value": iocs[channels_dict[cf_channel["name"]][-1]]["recordType"], - } + create_recordType_property( + owner, iocs[channels_dict[cf_channel["name"]][-1]]["recordType"] + ) ), - cf_channel["properties"], + cf_channel, + processor.managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) @@ -475,7 +515,7 @@ def __updateCF__( if alias["name"] in channels_dict: alias["owner"] = iocs[channels_dict[alias["name"]][-1]]["owner"] alias["properties"] = __merge_property_lists( - ch_create_properties( + create_default_properties( owner, iocTime, recceiverid, @@ -483,18 +523,19 @@ def __updateCF__( iocs, cf_channel, ), - alias["properties"], + alias, + processor.managed_properties, ) - if conf.get("recordType", "default") == "on": + if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( - { - "name": "recordType", - "owner": owner, - "value": iocs[channels_dict[alias["name"]][-1]]["recordType"], - } + create_recordType_property( + owner, + iocs[channels_dict[alias["name"]][-1]]["recordType"], + ) ), - cf_channel["properties"], + cf_channel, + processor.managed_properties, ) channels.append(alias) _log.debug("Add existing alias to previous IOC: {s}".format(s=channels[-1])) @@ -503,31 +544,23 @@ def __updateCF__( """Orphan the channel : mark as inactive, keep the old hostName and iocName""" cf_channel["properties"] = __merge_property_lists( [ - {"name": "pvStatus", "owner": owner, "value": "Inactive"}, - {"name": "time", "owner": owner, "value": iocTime}, + create_inactive_property(owner), + create_time_property(owner, iocTime), ], - cf_channel["properties"], + cf_channel, ) channels.append(cf_channel) _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) """Also orphan any alias""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: alias["properties"] = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Inactive", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, + create_inactive_property(owner), + create_time_property(owner, iocTime), ], - alias["properties"], + alias, ) channels.append(alias) _log.debug("Add orphaned alias with no IOC: {s}".format(s=channels[-1])) @@ -537,37 +570,36 @@ def __updateCF__( Channel exists in Channelfinder with same hostname and iocname. Update the status to ensure it is marked active and update the time. """ + _log.debug( + "Channel {s} exists in Channelfinder with same hostname and iocname".format( + s=cf_channel["name"] + ) + ) cf_channel["properties"] = __merge_property_lists( [ - {"name": "pvStatus", "owner": owner, "value": "Active"}, - {"name": "time", "owner": owner, "value": iocTime}, + create_active_property(owner), + create_time_property(owner, iocTime), ], - cf_channel["properties"], + cf_channel, + processor.managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel with same IOC: {s}".format(s=channels[-1])) new_channels.remove(cf_channel["name"]) """In case, alias exist""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: if alias in old_channels: """alias exists in old list""" alias["properties"] = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Active", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, + create_active_property(owner), + create_time_property(owner, iocTime), ], - alias["properties"], + alias, + processor.managed_properties, ) channels.append(alias) new_channels.remove(alias["name"]) @@ -575,30 +607,22 @@ def __updateCF__( """alias exists but not part of old list""" aprops = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Active", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, - { - "name": "alias", - "owner": owner, - "value": cf_channel["name"], - }, + create_active_property(owner), + create_time_property(owner, iocTime), + create_alias_property( + owner, + cf_channel["name"], + ), ], - cf_channel["properties"], + cf_channel, + processor.managed_properties, ) channels.append( - { - "name": alias["name"], - "owner": owner, - "properties": aprops, - } + create_channel( + alias["name"], + owner, + aprops, + ) ) new_channels.remove(alias["name"]) _log.debug("Add existing alias with same IOC: {s}".format(s=channels[-1])) @@ -632,45 +656,50 @@ def __updateCF__( for channel_name in new_channels: newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) - if conf.get("recordType", "default") == "on": - newProps.append( - { - "name": "recordType", - "owner": owner, - "value": recordInfoByName[channel_name]["recordType"], - } - ) + if conf.get("recordType"): + newProps.append(create_recordType_property(owner, recordInfoByName[channel_name]["recordType"])) if channel_name in recordInfoByName and "infoProperties" in recordInfoByName[channel_name]: newProps = newProps + recordInfoByName[channel_name]["infoProperties"] if channel_name in existingChannels: - """update existing channel: exists but with a different hostName and/or iocName""" + _log.debug( + f"""update existing channel{channel_name}: exists but with a different hostName and/or iocName""" + ) + existingChannel = existingChannels[channel_name] - existingChannel["properties"] = __merge_property_lists(newProps, existingChannel["properties"]) + existingChannel["properties"] = __merge_property_lists( + newProps, + existingChannel, + processor.managed_properties, + ) channels.append(existingChannel) _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: - alProps = [{"name": "alias", "owner": owner, "value": channel_name}] + alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name]["aliases"]: if alias in existingChannels: ach = existingChannels[alias] - ach["properties"] = __merge_property_lists(alProps, ach["properties"]) + ach["properties"] = __merge_property_lists( + alProps, + ach, + processor.managed_properties, + ) channels.append(ach) else: - channels.append({"name": alias, "owner": owner, "properties": alProps}) + channels.append(create_channel(alias, owner, alProps)) _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) else: """New channel""" channels.append({"name": channel_name, "owner": owner, "properties": newProps}) _log.debug("Add new channel: {s}".format(s=channels[-1])) - if conf.get("alias", "default") == "on": + if conf.get("alias"): if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: - alProps = [{"name": "alias", "owner": owner, "value": channel_name}] + alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name]["aliases"]: @@ -688,17 +717,17 @@ def __updateCF__( def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid): return [ - {"name": "hostName", "owner": owner, "value": hostName}, - {"name": "iocName", "owner": owner, "value": iocName}, - {"name": "iocid", "owner": owner, "value": iocid}, - {"name": "iocIP", "owner": owner, "value": iocIP}, - {"name": "pvStatus", "owner": owner, "value": "Active"}, - {"name": "time", "owner": owner, "value": iocTime}, - {"name": RECCEIVERID_KEY, "owner": owner, "value": recceiverid}, + create_property(owner, "hostName", hostName), + create_property(owner, "iocName", iocName), + create_property(owner, "iocid", iocid), + create_property(owner, "iocIP", iocIP), + create_active_property(owner), + create_time_property(owner, iocTime), + create_property(owner, RECCEIVERID_KEY, recceiverid), ] -def ch_create_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel): +def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel): return create_properties( owner, iocTime, @@ -710,15 +739,17 @@ def ch_create_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_ch ) -def __merge_property_lists(newProperties, oldProperties): +def __merge_property_lists( + newProperties: list[dict[str, str]], channel: dict[str, list[dict[str, str]]], managed_properties=set() +) -> list[dict[str, str]]: """ Merges two lists of properties ensuring that there are no 2 properties with the same name In case of overlap between the new and old property lists the new property list wins out """ newPropNames = [p["name"] for p in newProperties] - for oldProperty in oldProperties: - if oldProperty["name"] not in newPropNames: + for oldProperty in channel["properties"]: + if oldProperty["name"] not in newPropNames and (oldProperty["name"] not in managed_properties): newProperties = newProperties + [oldProperty] return newProperties diff --git a/server/recceiver/recast.py b/server/recceiver/recast.py index 3c0c6e30..b0abf6b4 100644 --- a/server/recceiver/recast.py +++ b/server/recceiver/recast.py @@ -241,6 +241,18 @@ def __str__(self): source_address, init, conn, nenv, nadd, nalias, ninfo, ndel ) + def __repr__(self): + return f"""Transaction( + source_address={self.source_address}, + initial={self.initial}, + connected={self.connected}, + records_to_add={self.records_to_add}, + client_infos={self.client_infos}, + record_infos_to_add={self.record_infos_to_add}, + aliases={self.aliases}, + records_to_delete={self.records_to_delete}) + """ + class CollectionSession(object): timeout = 5.0 diff --git a/server/test-compose.yml b/server/test-compose.yml deleted file mode 100644 index 0a7b264e..00000000 --- a/server/test-compose.yml +++ /dev/null @@ -1,156 +0,0 @@ -services: - cf: - image: ghcr.io/channelfinder/channelfinderservice:master - hostname: cf - networks: - - net-2-cf - ports: - - "8080:8080" - depends_on: - elasticsearch: - condition: service_healthy - restart: true - environment: - ELASTICSEARCH_NETWORK_HOST: elasticsearch-cf - ELASTICSEARCH_QUERY_SIZE: 10000 - demo_auth.enabled: true - demo_auth.users: admin - demo_auth.pwds: password - demo_auth.roles: ADMIN - EPICS_PVAS_INTF_ADDR_LIST: "0.0.0.0" - aa.enabled: false - logging.level.org.springframework.web: INFO - healthcheck: - test: curl -s -f http://cf:8080/ChannelFinder - interval: 10s - timeout: 60s - retries: 5 - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4 - hostname: elasticsearch-cf - networks: - - net-2-cf - environment: - cluster.name: channelfinder - discovery.type: single-node - bootstrap.memory_lock: "true" - xpack.security.enabled: "false" - logger.level: WARN - healthcheck: - test: curl -s -f http://localhost:9200/_cluster/health - interval: 10s - timeout: 60s - retries: 5 - volumes: - - channelfinder-es-data:/usr/share/elasticsearch/data - - recc1: - build: . - hostname: recc1 - networks: - - net-0-recc-1 - - net-2-cf - depends_on: - cf: - condition: service_healthy - restart: true - healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 - volumes: - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/cf.conf - read_only: true - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/channelfinderapi.conf - read_only: true - recc2: - build: . - hostname: recc2 - networks: - - net-1-recc-2 - - net-2-cf - depends_on: - cf: - condition: service_healthy - restart: true - healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 - volumes: - - type: bind - source: docker/config/cf2.conf - target: /home/recceiver/cf.conf - read_only: true - - type: bind - source: docker/config/cf2.conf - target: /home/recceiver/channelfinderapi.conf - read_only: true - - ioc1-1: - build: ../client - hostname: ioc1-1 - tty: true - environment: - - IOCSH_NAME=IOC1-1 - networks: - - net-0-recc-1 - depends_on: - recc1: - condition: service_healthy - restart: true - ioc1-2: - build: ../client - hostname: ioc1-2 - tty: true - environment: - - IOCSH_NAME=IOC1-2 - networks: - - net-0-recc-1 - depends_on: - recc2: - condition: service_healthy - restart: true - ioc2-1: - build: ../client - hostname: ioc2-1 - tty: true - environment: - - IOCSH_NAME=IOC2-1 - networks: - - net-1-recc-2 - depends_on: - cf: - condition: service_healthy - restart: true - ioc2-2: - build: ../client - hostname: ioc2-2 - tty: true - environment: - - IOCSH_NAME=IOC2-2 - networks: - - net-1-recc-2 - depends_on: - cf: - condition: service_healthy - restart: true - -volumes: - channelfinder-es-data: - driver: local - -networks: - net-2-cf: - driver: bridge - net-0-recc-1: - driver: bridge - net-1-recc-2: - driver: bridge diff --git a/server/tests/__init__.py b/server/tests/__init__.py index e69de29b..795a66e9 100644 --- a/server/tests/__init__.py +++ b/server/tests/__init__.py @@ -0,0 +1,7 @@ +import logging + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + encoding="utf-8", +) diff --git a/server/tests/client_checks.py b/server/tests/client_checks.py new file mode 100644 index 00000000..9af7ce0e --- /dev/null +++ b/server/tests/client_checks.py @@ -0,0 +1,70 @@ +import logging +import time +from typing import Callable + +from channelfinder import ChannelFinderClient +from testcontainers.compose import DockerCompose + +LOG: logging.Logger = logging.getLogger(__name__) + +ACTIVE_PROPERTY = {"name": "pvStatus", "owner": "admin", "value": "Active", "channels": []} +INACTIVE_PROPERTY = {"name": "pvStatus", "owner": "admin", "value": "Inactive", "channels": []} +MAX_WAIT_SECONDS = 180 +TIME_PERIOD_INCREMENT = 2 + + +def channel_match(channel0, channel1, properties_to_match: list[str]): + assert channel0["name"] == channel1["name"] + assert channel0["owner"] == channel1["owner"] + + for prop in channel0["properties"]: + assert not (prop["name"] in properties_to_match and prop not in channel1["properties"]), ( + f"Property {prop} not found in channel {channel1['name']}" + ) + + +def channels_match(channels_begin, channels_end, properties_to_match: list[str]): + for channel_index, channel in enumerate(channels_begin): + channel_match(channel, channels_end[channel_index], properties_to_match) + + +def check_channel_count(cf_client: ChannelFinderClient, name="*", expected_channel_count=24): + channels = cf_client.find(name=name) + LOG.debug("Found %s channels", len(channels)) + return len(channels) == expected_channel_count + + +def check_channel_property(cf_client: ChannelFinderClient, name="*", prop=ACTIVE_PROPERTY): + channels = cf_client.find(name=name) + active_channels = (prop in channel["properties"] for channel in channels) + return all(active_channels) + + +def wait_for_sync(cf_client: ChannelFinderClient, check: Callable[[ChannelFinderClient], bool]) -> bool: + time_period_to_wait_seconds = 1 + total_seconds_waited = 0 + while total_seconds_waited < MAX_WAIT_SECONDS: + if check(cf_client): + return True + time.sleep(time_period_to_wait_seconds) + total_seconds_waited += time_period_to_wait_seconds + time_period_to_wait_seconds += TIME_PERIOD_INCREMENT + return False + + +def create_client_and_wait(compose: DockerCompose, expected_channel_count=24) -> ChannelFinderClient: + LOG.info("Waiting for channels to sync") + cf_client = create_client_from_compose(compose) + assert wait_for_sync( + cf_client, lambda cf_client: check_channel_count(cf_client, expected_channel_count=expected_channel_count) + ) + return cf_client + + +def create_client_from_compose(compose: DockerCompose) -> ChannelFinderClient: + cf_host, cf_port = compose.get_service_host_and_port("cf") + cf_url = f"http://{cf_host if cf_host else 'localhost'}:{cf_port}/ChannelFinder" + # wait for channels to sync + LOG.info("CF URL: %s", cf_url) + cf_client = ChannelFinderClient(BaseURL=cf_url, username="admin", password="password") + return cf_client diff --git a/server/tests/docker.py b/server/tests/docker.py index 35fe2940..455cf7a9 100644 --- a/server/tests/docker.py +++ b/server/tests/docker.py @@ -1,5 +1,5 @@ import logging -import pathlib +from pathlib import Path import pytest from testcontainers.compose import DockerCompose @@ -9,12 +9,12 @@ LOG: logging.Logger = logging.getLogger(__name__) -def test_compose() -> DockerCompose: - current_path = pathlib.Path(__file__).parent.resolve() +def test_compose(compose_file=Path("docker") / Path("test-multi-recc.yml")) -> DockerCompose: + current_path = Path(__file__).parent.resolve() return DockerCompose( str(current_path.parent.resolve()), - compose_file_name=str(current_path.parent.joinpath("test-compose.yml").resolve()), + compose_file_name=str(current_path.parent.joinpath(compose_file).resolve()), build=True, ) @@ -40,3 +40,10 @@ def setup_compose(): if LOG.level <= logging.DEBUG: fetch_containers_and_log_logs(compose) compose.stop() + + +def restart_container(compose: DockerCompose, host_name: str) -> None: + container = compose.get_container(host_name) + docker_client = DockerClient() + docker_client.containers.get(container.ID).stop() + docker_client.containers.get(container.ID).start() diff --git a/server/tests/test_e2e.py b/server/tests/test_e2e.py index 66bdb68a..3ba10435 100644 --- a/server/tests/test_e2e.py +++ b/server/tests/test_e2e.py @@ -1,73 +1,32 @@ import logging -import time import pytest from channelfinder import ChannelFinderClient +from .client_checks import create_client_and_wait from .docker import setup_compose # noqa: F401 -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - encoding="utf-8", -) - LOG: logging.Logger = logging.getLogger(__name__) -MAX_WAIT_SECONDS = 180 -TIME_PERIOD_INCREMENT = 2 - +RECSYNC_RESTART_DELAY = 30 # Number of channels expected in the default setup # 4 iocs, 6 channels per ioc (2 reccaster.db, 2 somerecords.db, 2 aliases in somerecords.db) -EXPECTED_DEFAULT_CHANNELS = 24 - - -def check_channel_count(cf_client, name="*"): - channels = cf_client.find(name=name) - LOG.debug("Found %s channels", len(channels)) - return len(channels) == EXPECTED_DEFAULT_CHANNELS - - -def wait_for_sync(cf_client, check): - time_period_to_wait_seconds = 1 - total_seconds_waited = 0 - while total_seconds_waited < MAX_WAIT_SECONDS: - if check(cf_client): - break - time.sleep(time_period_to_wait_seconds) - total_seconds_waited += time_period_to_wait_seconds - time_period_to_wait_seconds += TIME_PERIOD_INCREMENT - - -def create_client_and_wait(compose): - LOG.info("Waiting for channels to sync") - cf_client = create_client_from_compose(compose) - wait_for_sync(cf_client, lambda cf_client: check_channel_count(cf_client)) - return cf_client - - -def create_client_from_compose(compose): - cf_host, cf_port = compose.get_service_host_and_port("cf") - cf_url = f"http://{cf_host if cf_host else 'localhost'}:{cf_port}/ChannelFinder" - # wait for channels to sync - LOG.info("CF URL: %s", cf_url) - cf_client = ChannelFinderClient(BaseURL=cf_url) - return cf_client +EXPECTED_DEFAULT_CHANNEL_COUNT = 32 @pytest.fixture(scope="class") def cf_client(setup_compose): # noqa: F811 - return create_client_and_wait(setup_compose) + return create_client_and_wait(setup_compose, EXPECTED_DEFAULT_CHANNEL_COUNT) class TestE2E: - def test_number_of_channels_and_channel_name(self, cf_client) -> None: + def test_number_of_channels_and_channel_name(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(name="*") - assert len(channels) == EXPECTED_DEFAULT_CHANNELS + assert len(channels) == EXPECTED_DEFAULT_CHANNEL_COUNT assert channels[0]["name"] == "IOC1-1:Msg-I" # Smoke Test Default Properties - def test_number_of_aliases_and_alais_property(self, cf_client) -> None: + def test_number_of_aliases_and_alais_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("alias", "*")]) assert len(channels) == 8 assert channels[0]["name"] == "IOC1-1:lix1" @@ -78,7 +37,7 @@ def test_number_of_aliases_and_alais_property(self, cf_client) -> None: "channels": [], } in channels[0]["properties"] - def test_number_of_recordDesc_and_property(self, cf_client) -> None: + def test_number_of_recordDesc_and_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("recordDesc", "*")]) assert len(channels) == 4 assert { @@ -88,9 +47,9 @@ def test_number_of_recordDesc_and_property(self, cf_client) -> None: "channels": [], } in channels[0]["properties"] - def test_number_of_recordType_and_property(self, cf_client) -> None: + def test_number_of_recordType_and_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("recordType", "*")]) - assert len(channels) == EXPECTED_DEFAULT_CHANNELS + assert len(channels) == EXPECTED_DEFAULT_CHANNEL_COUNT assert { "name": "recordType", "value": "stringin", diff --git a/server/tests/test_remove_property.py b/server/tests/test_remove_property.py new file mode 100644 index 00000000..7b5950d3 --- /dev/null +++ b/server/tests/test_remove_property.py @@ -0,0 +1,94 @@ +import logging +import threading +from pathlib import Path + +import pytest +from testcontainers.compose import DockerCompose + +from docker import DockerClient +from docker.models.containers import Container + +from .client_checks import INACTIVE_PROPERTY, check_channel_property, create_client_and_wait, wait_for_sync +from .docker import fetch_containers_and_log_logs, test_compose # noqa: F401 + +LOG: logging.Logger = logging.getLogger(__name__) + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + encoding="utf-8", +) + + +@pytest.fixture(scope="class") +def setup_compose(): + LOG.info("Setup remove test environment") + compose = test_compose(Path("docker") / Path("test-remove-infotag.yml")) + compose.start() + yield compose + LOG.info("Teardown test environment") + LOG.info("Stopping docker compose") + if LOG.level <= logging.DEBUG: + fetch_containers_and_log_logs(compose) + compose.stop() + + +def docker_exec_new_command(container: Container, command: str): + def stream_logs(exec_result, cmd: str): + if LOG.level <= logging.DEBUG: + LOG.debug("Logs from %s with command %s", container.name, cmd) + for line in exec_result.output: + LOG.debug(line.decode().strip()) + + exec_result = container.exec_run(command, tty=True, stream=True) + log_thread = threading.Thread( + target=stream_logs, + args=( + exec_result, + command, + ), + ) + log_thread.start() + + +class TestRemoveProperty: + def test_remove_property(self, setup_compose: DockerCompose) -> None: # noqa: F811 + """ + Test that the setup in the docker compose creates channels in channelfinder + """ + ioc_container = setup_compose.get_container("ioc1-1") + docker_client = DockerClient() + docker_ioc = docker_client.containers.get(ioc_container.ID) + docker_exec_new_command(docker_ioc, "./demo /recsync/iocBoot/iocdemo/st.cmd") + + LOG.info("Waiting for channels to sync") + cf_client = create_client_and_wait(setup_compose, expected_channel_count=8) + + # Check ioc1-1 has ai:archive with info tag "archive" + LOG.debug('Checking ioc1-1 has ai:archive with info tag "archive"') + archive_channel_name = "IOC1-1:ai:archive" + archive_channel = cf_client.find(name=archive_channel_name) + + def get_len_archive_properties(archive_channel): + return len([prop for prop in archive_channel[0]["properties"] if prop["name"] == "archive"]) + + assert get_len_archive_properties(archive_channel) == 1 + + docker_ioc.stop() + LOG.info("Waiting for channels to go inactive") + assert wait_for_sync( + cf_client, + lambda cf_client: check_channel_property(cf_client, name=archive_channel_name, prop=INACTIVE_PROPERTY), + ) + docker_ioc.start() + + docker_exec_new_command(docker_ioc, "./demo /recsync/iocBoot/iocdemo/st_bugtest.cmd") + # Detach by not waiting for the thread to finish + + LOG.debug("ioc1-1 restart") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, name=archive_channel_name)) + LOG.debug("ioc1-1 has restarted and synced") + + archive_channel = cf_client.find(name=archive_channel_name) + LOG.debug("archive channel: %s", archive_channel) + assert get_len_archive_properties(archive_channel) == 0 diff --git a/server/tests/test_restart.py b/server/tests/test_restart.py new file mode 100644 index 00000000..b7fbc4b1 --- /dev/null +++ b/server/tests/test_restart.py @@ -0,0 +1,47 @@ +import logging + +import pytest +from channelfinder import ChannelFinderClient +from testcontainers.compose import DockerCompose + +from .client_checks import channels_match, check_channel_property, create_client_and_wait, wait_for_sync +from .docker import restart_container, setup_compose # noqa: F401 + +PROPERTIES_TO_MATCH = ["pvStatus", "recordType", "recordDesc", "alias", "hostName", "iocName", "recceiverID"] + +LOG: logging.Logger = logging.getLogger(__name__) + +EXPECTED_DEFAULT_CHANNEL_COUNT = 32 + + +@pytest.fixture(scope="class") +def cf_client(setup_compose): # noqa: F811 + return create_client_and_wait(setup_compose, expected_channel_count=EXPECTED_DEFAULT_CHANNEL_COUNT) + + +class TestRestartIOC: + def test_channels_same_after_restart(self, setup_compose: DockerCompose, cf_client: ChannelFinderClient) -> None: # noqa: F811 + channels_begin = cf_client.find(name="*") + restart_container(setup_compose, "ioc1-1") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, "IOC1-1:Msg-I")) + channels_end = cf_client.find(name="*") + assert len(channels_begin) == len(channels_end) + channels_match(channels_begin, channels_end, PROPERTIES_TO_MATCH) + + def test_manual_channels_same_after_restart( + self, + setup_compose: DockerCompose, # noqa: F811 + cf_client: ChannelFinderClient, + ) -> None: + test_property = {"name": "test_property", "owner": "testowner"} + cf_client.set(properties=[test_property]) + test_property_value = test_property | {"value": "test_value"} + channels = cf_client.find(name="IOC1-1:Msg-I") + channels[0]["properties"] = [test_property_value] + cf_client.set(property=test_property) + channels_begin = cf_client.find(name="*") + restart_container(setup_compose, "ioc1-1") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, "IOC1-1:Msg-I")) + channels_end = cf_client.find(name="*") + assert len(channels_begin) == len(channels_end) + channels_match(channels_begin, channels_end, PROPERTIES_TO_MATCH + ["test_property"])