From 7be892a2aed2d04b5a6d7424219635ffd098296b Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 13 Mar 2025 13:27:26 +0100 Subject: [PATCH 01/10] Use a create_property method --- server/recceiver/cfstore.py | 162 ++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ce23e6f2..49d414ed 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -231,12 +231,9 @@ def _commitWithThread(self, transaction): if recinfo_wl: recordInfo[record_id]["infoProperties"] = list() for infotag in recinfo_wl: - property = { - "name": infotag, - "owner": owner, - "value": record_infos_to_add[infotag], - } - recordInfo[record_id]["infoProperties"].append(property) + recordInfo[record_id]["infoProperties"].append( + create_property(owner, infotag, record_infos_to_add[infotag]) + ) for record_id, alias in transaction.aliases.items(): if record_id not in recordInfo: @@ -251,14 +248,11 @@ def _commitWithThread(self, transaction): for record_id in recordInfo: for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: - property = { - "name": cf_prop_name, - "owner": owner, - "value": transaction.client_infos.get(epics_env_var_name), - } if "infoProperties" not in recordInfo[record_id]: recordInfo[record_id]["infoProperties"] = list() - recordInfo[record_id]["infoProperties"].append(property) + recordInfo[record_id]["infoProperties"].append( + create_property(owner, cf_prop_name, transaction.client_infos.get(epics_env_var_name)) + ) else: _log.debug( "EPICS environment var %s listed in environment_vars setting list not found in this IOC: %s", @@ -381,7 +375,7 @@ def clean_channels(self, owner, channels): 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) ) self.client.update( - property={"name": "pvStatus", "owner": owner, "value": "Inactive"}, + property=create_inactive_property(owner), channelNames=new_channels, ) @@ -401,6 +395,46 @@ def dict_to_file(dict, iocs, conf): json.dump(list, f) +def create_channel(name: str, owner: str, properties: list[dict[str, str]]): + return { + "name": name, + "owner": owner, + "properties": properties, + } + + +def create_property(owner: str, name: str, value: str): + return { + "name": name, + "owner": owner, + "value": value, + } + + +def create_recordType_property(owner: str, recordType: str): + return create_property(owner, "recordType", recordType) + + +def create_alias_property(owner: str, alias: str): + return create_property(owner, "alias", alias) + + +def create_pvStatus_property(owner: str, pvStatus: str): + return create_property(owner, "pvStatus", pvStatus) + + +def create_active_property(owner: str): + return create_pvStatus_property(owner, "Active") + + +def create_inactive_property(owner: str): + return create_pvStatus_property(owner, "Inactive") + + +def create_time_property(owner: str, time: str): + return create_property(owner, "time", time) + + def __updateCF__( processor, recordInfoByName, @@ -452,17 +486,15 @@ def __updateCF__( if cf_channel["name"] in channels_dict: cf_channel["owner"] = iocs[channels_dict[cf_channel["name"]][-1]]["owner"] cf_channel["properties"] = __merge_property_lists( - ch_create_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), + create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), cf_channel["properties"], ) if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( - { - "name": "recordType", - "owner": owner, - "value": iocs[channels_dict[cf_channel["name"]][-1]]["recordType"], - } + create_recordType_property( + owner, iocs[channels_dict[cf_channel["name"]][-1]]["recordType"] + ) ), cf_channel["properties"], ) @@ -475,7 +507,7 @@ def __updateCF__( if alias["name"] in channels_dict: alias["owner"] = iocs[channels_dict[alias["name"]][-1]]["owner"] alias["properties"] = __merge_property_lists( - ch_create_properties( + create_default_properties( owner, iocTime, recceiverid, @@ -488,11 +520,10 @@ def __updateCF__( if conf.get("recordType", "default") == "on": cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( - { - "name": "recordType", - "owner": owner, - "value": iocs[channels_dict[alias["name"]][-1]]["recordType"], - } + create_recordType_property( + owner, + iocs[channels_dict[alias["name"]][-1]]["recordType"], + ) ), cf_channel["properties"], ) @@ -503,8 +534,8 @@ def __updateCF__( """Orphan the channel : mark as inactive, keep the old hostName and iocName""" cf_channel["properties"] = __merge_property_lists( [ - {"name": "pvStatus", "owner": owner, "value": "Inactive"}, - {"name": "time", "owner": owner, "value": iocTime}, + create_inactive_property(owner), + create_time_property(owner, iocTime), ], cf_channel["properties"], ) @@ -516,16 +547,8 @@ def __updateCF__( for alias in recordInfoByName[cf_channel["name"]]["aliases"]: alias["properties"] = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Inactive", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, + create_inactive_property(owner), + create_time_property(owner, iocTime), ], alias["properties"], ) @@ -539,8 +562,8 @@ def __updateCF__( """ cf_channel["properties"] = __merge_property_lists( [ - {"name": "pvStatus", "owner": owner, "value": "Active"}, - {"name": "time", "owner": owner, "value": iocTime}, + create_active_property(owner), + create_time_property(owner, iocTime), ], cf_channel["properties"], ) @@ -556,16 +579,8 @@ def __updateCF__( """alias exists in old list""" alias["properties"] = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Active", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, + create_active_property(owner), + create_time_property(owner, iocTime), ], alias["properties"], ) @@ -575,21 +590,12 @@ def __updateCF__( """alias exists but not part of old list""" aprops = __merge_property_lists( [ - { - "name": "pvStatus", - "owner": owner, - "value": "Active", - }, - { - "name": "time", - "owner": owner, - "value": iocTime, - }, - { - "name": "alias", - "owner": owner, - "value": cf_channel["name"], - }, + create_active_property(owner), + create_time_property(owner, iocTime), + create_alias_property( + owner, + cf_channel["name"], + ), ], cf_channel["properties"], ) @@ -633,13 +639,7 @@ def __updateCF__( for channel_name in new_channels: newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) if conf.get("recordType", "default") == "on": - newProps.append( - { - "name": "recordType", - "owner": owner, - "value": recordInfoByName[channel_name]["recordType"], - } - ) + newProps.append(create_recordType_property(owner, recordInfoByName[channel_name]["recordType"])) if channel_name in recordInfoByName and "infoProperties" in recordInfoByName[channel_name]: newProps = newProps + recordInfoByName[channel_name]["infoProperties"] @@ -652,7 +652,7 @@ def __updateCF__( """in case, alias exists, update their properties too""" if conf.get("alias", "default") == "on": if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: - alProps = [{"name": "alias", "owner": owner, "value": channel_name}] + alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name]["aliases"]: @@ -670,7 +670,7 @@ def __updateCF__( _log.debug("Add new channel: {s}".format(s=channels[-1])) if conf.get("alias", "default") == "on": if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: - alProps = [{"name": "alias", "owner": owner, "value": channel_name}] + alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name]["aliases"]: @@ -688,17 +688,17 @@ def __updateCF__( def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid): return [ - {"name": "hostName", "owner": owner, "value": hostName}, - {"name": "iocName", "owner": owner, "value": iocName}, - {"name": "iocid", "owner": owner, "value": iocid}, - {"name": "iocIP", "owner": owner, "value": iocIP}, - {"name": "pvStatus", "owner": owner, "value": "Active"}, - {"name": "time", "owner": owner, "value": iocTime}, - {"name": RECCEIVERID_KEY, "owner": owner, "value": recceiverid}, + create_property(owner, "hostName", hostName), + create_property(owner, "iocName", iocName), + create_property(owner, "iocid", iocid), + create_property(owner, "iocIP", iocIP), + create_active_property(owner), + create_time_property(owner, iocTime), + create_property(owner, RECCEIVERID_KEY, recceiverid), ] -def ch_create_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel): +def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel): return create_properties( owner, iocTime, From 705263c41138397a93378c172b635a16b70e5054 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 13 Mar 2025 14:22:36 +0100 Subject: [PATCH 02/10] Use a create_channel method --- server/recceiver/cfstore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 49d414ed..2965ba6f 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -600,11 +600,11 @@ def __updateCF__( cf_channel["properties"], ) channels.append( - { - "name": alias["name"], - "owner": owner, - "properties": aprops, - } + create_channel( + alias["name"], + owner, + aprops, + ) ) new_channels.remove(alias["name"]) _log.debug("Add existing alias with same IOC: {s}".format(s=channels[-1])) @@ -661,7 +661,7 @@ def __updateCF__( ach["properties"] = __merge_property_lists(alProps, ach["properties"]) channels.append(ach) else: - channels.append({"name": alias, "owner": owner, "properties": alProps}) + channels.append(create_channel(alias, owner, alProps)) _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) else: From b90306adc115b65477ce370df10e1ef18e1aabe1 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 13 Mar 2025 14:43:09 +0100 Subject: [PATCH 03/10] update merge_property_lists to take a channel --- server/recceiver/cfstore.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 2965ba6f..067aab67 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -487,7 +487,7 @@ def __updateCF__( cf_channel["owner"] = iocs[channels_dict[cf_channel["name"]][-1]]["owner"] cf_channel["properties"] = __merge_property_lists( create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), - cf_channel["properties"], + cf_channel, ) if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( @@ -496,7 +496,7 @@ def __updateCF__( owner, iocs[channels_dict[cf_channel["name"]][-1]]["recordType"] ) ), - cf_channel["properties"], + cf_channel, ) channels.append(cf_channel) _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) @@ -515,7 +515,7 @@ def __updateCF__( iocs, cf_channel, ), - alias["properties"], + alias, ) if conf.get("recordType", "default") == "on": cf_channel["properties"] = __merge_property_lists( @@ -525,7 +525,7 @@ def __updateCF__( iocs[channels_dict[alias["name"]][-1]]["recordType"], ) ), - cf_channel["properties"], + cf_channel, ) channels.append(alias) _log.debug("Add existing alias to previous IOC: {s}".format(s=channels[-1])) @@ -537,7 +537,7 @@ def __updateCF__( create_inactive_property(owner), create_time_property(owner, iocTime), ], - cf_channel["properties"], + cf_channel, ) channels.append(cf_channel) _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) @@ -550,7 +550,7 @@ def __updateCF__( create_inactive_property(owner), create_time_property(owner, iocTime), ], - alias["properties"], + alias, ) channels.append(alias) _log.debug("Add orphaned alias with no IOC: {s}".format(s=channels[-1])) @@ -565,7 +565,7 @@ def __updateCF__( create_active_property(owner), create_time_property(owner, iocTime), ], - cf_channel["properties"], + cf_channel, ) channels.append(cf_channel) _log.debug("Add existing channel with same IOC: {s}".format(s=channels[-1])) @@ -582,7 +582,7 @@ def __updateCF__( create_active_property(owner), create_time_property(owner, iocTime), ], - alias["properties"], + alias, ) channels.append(alias) new_channels.remove(alias["name"]) @@ -597,7 +597,7 @@ def __updateCF__( cf_channel["name"], ), ], - cf_channel["properties"], + cf_channel, ) channels.append( create_channel( @@ -646,7 +646,7 @@ def __updateCF__( if channel_name in existingChannels: """update existing channel: exists but with a different hostName and/or iocName""" existingChannel = existingChannels[channel_name] - existingChannel["properties"] = __merge_property_lists(newProps, existingChannel["properties"]) + existingChannel["properties"] = __merge_property_lists(newProps, existingChannel) channels.append(existingChannel) _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" @@ -658,7 +658,7 @@ def __updateCF__( for alias in recordInfoByName[channel_name]["aliases"]: if alias in existingChannels: ach = existingChannels[alias] - ach["properties"] = __merge_property_lists(alProps, ach["properties"]) + ach["properties"] = __merge_property_lists(alProps, ach) channels.append(ach) else: channels.append(create_channel(alias, owner, alProps)) @@ -710,14 +710,14 @@ def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, ) -def __merge_property_lists(newProperties, oldProperties): +def __merge_property_lists(newProperties: list[dict[str, str]], channel: dict[str, list[dict[str, str]]]): """ Merges two lists of properties ensuring that there are no 2 properties with the same name In case of overlap between the new and old property lists the new property list wins out """ newPropNames = [p["name"] for p in newProperties] - for oldProperty in oldProperties: + for oldProperty in channel["properties"]: if oldProperty["name"] not in newPropNames: newProperties = newProperties + [oldProperty] return newProperties From 08c366cd9fdbdce15ef009af8edce27ff057c039 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 13 Mar 2025 14:54:51 +0100 Subject: [PATCH 04/10] Replace "default") == "on" with just ) --- server/recceiver/cfstore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 067aab67..c31e867f 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -517,7 +517,7 @@ def __updateCF__( ), alias, ) - if conf.get("recordType", "default") == "on": + if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( create_recordType_property( @@ -542,7 +542,7 @@ def __updateCF__( channels.append(cf_channel) _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) """Also orphan any alias""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: alias["properties"] = __merge_property_lists( @@ -572,7 +572,7 @@ def __updateCF__( new_channels.remove(cf_channel["name"]) """In case, alias exist""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: if alias in old_channels: @@ -638,7 +638,7 @@ def __updateCF__( for channel_name in new_channels: newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) - if conf.get("recordType", "default") == "on": + if conf.get("recordType"): newProps.append(create_recordType_property(owner, recordInfoByName[channel_name]["recordType"])) if channel_name in recordInfoByName and "infoProperties" in recordInfoByName[channel_name]: newProps = newProps + recordInfoByName[channel_name]["infoProperties"] @@ -650,7 +650,7 @@ def __updateCF__( channels.append(existingChannel) _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" - if conf.get("alias", "default") == "on": + if conf.get("alias"): if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: alProps = [create_alias_property(owner, channel_name)] for p in newProps: @@ -668,7 +668,7 @@ def __updateCF__( """New channel""" channels.append({"name": channel_name, "owner": owner, "properties": newProps}) _log.debug("Add new channel: {s}".format(s=channels[-1])) - if conf.get("alias", "default") == "on": + if conf.get("alias"): if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: alProps = [create_alias_property(owner, channel_name)] for p in newProps: From 0c2784e0234ab652752d0108f853cd2b3c4475f9 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 25 Feb 2025 14:05:13 +0100 Subject: [PATCH 05/10] Automate testing info tag removal bug in recceiver Bug can be reproduced manually as follows 1. Create recceiver with infotag (X) set to be updated in channelfinder. 2. Add IOC with PV with infotag X set to A 3. Channelfinder has PV with X=A 4. Reboot recceiver with infotag X removed 5. Channelfinder has PV with X=A still even after IOC update Or by 1. Create recceiver with infotag (X) set to be updated in channelfinder. 2. Add IOC with PV with infotag X set to A 3. Channelfinder has PV with X=A 4. Reboot IOC with PV with infotag X removed 5. Channelfinder has PV with X=A still --- client/.dockerignore | 1 + client/Dockerfile | 5 +- client/demoApp/Db/Makefile | 3 +- client/demoApp/Db/archive_bugtest.db | 10 +++ client/iocBoot/iocdemo/st.cmd | 1 + client/iocBoot/iocdemo/st_bugtest.cmd | 31 +++++++++ server/test-compose.yml | 7 +- server/test-remove-compose.yml | 88 +++++++++++++++++++++++++ server/tests/__init__.py | 7 ++ server/tests/client_checks.py | 71 ++++++++++++++++++++ server/tests/docker.py | 11 +++- server/tests/test_e2e.py | 61 +++--------------- server/tests/test_remove_property.py | 93 +++++++++++++++++++++++++++ server/tests/test_restart.py | 47 ++++++++++++++ 14 files changed, 374 insertions(+), 62 deletions(-) create mode 100644 client/demoApp/Db/archive_bugtest.db create mode 100644 client/iocBoot/iocdemo/st_bugtest.cmd create mode 100644 server/test-remove-compose.yml create mode 100644 server/tests/client_checks.py create mode 100644 server/tests/test_remove_property.py create mode 100644 server/tests/test_restart.py diff --git a/client/.dockerignore b/client/.dockerignore index baca19af..44d0fd3a 100644 --- a/client/.dockerignore +++ b/client/.dockerignore @@ -13,3 +13,4 @@ configure/*.local *~ .*.swp +.ci diff --git a/client/Dockerfile b/client/Dockerfile index 714f2d8d..3aa7c419 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -50,8 +50,9 @@ RUN mv docker/RELEASE.local configure/RELEASE.local ENV EPICS_ROOT=/epics ENV EPICS_BASE=${EPICS_ROOT}/base RUN make -WORKDIR /recsync/iocBoot/iocdemo FROM recsync-base AS ioc-runner -CMD /recsync/bin/${EPICS_HOST_ARCH}/demo st.cmd +WORKDIR /recsync/bin/${EPICS_HOST_ARCH} + +CMD ./demo /recsync/iocBoot/iocdemo/st.cmd diff --git a/client/demoApp/Db/Makefile b/client/demoApp/Db/Makefile index 309d4854..60db2767 100644 --- a/client/demoApp/Db/Makefile +++ b/client/demoApp/Db/Makefile @@ -11,7 +11,8 @@ include $(TOP)/configure/CONFIG # Create and install (or just install) into /db # databases, templates, substitutions like this DB += somerecords.db - +DB += archive.db +DB += archive_bugtest.db #---------------------------------------------------- # If .db template is not named *.template add # _template = diff --git a/client/demoApp/Db/archive_bugtest.db b/client/demoApp/Db/archive_bugtest.db new file mode 100644 index 00000000..833e0bc6 --- /dev/null +++ b/client/demoApp/Db/archive_bugtest.db @@ -0,0 +1,10 @@ + +record(ai, "$(P)ai:archive") { + info("test", "testing") +} + +record(longout, "$(P)lo:archive") { + info("test", "testing") + info("hello", "world") + info("archive", "default") +} diff --git a/client/iocBoot/iocdemo/st.cmd b/client/iocBoot/iocdemo/st.cmd index 071829fa..fe81b085 100755 --- a/client/iocBoot/iocdemo/st.cmd +++ b/client/iocBoot/iocdemo/st.cmd @@ -26,5 +26,6 @@ addReccasterEnvVars("BUILDING") ## Load record instances dbLoadRecords("../../db/reccaster.db", "P=$(IOCSH_NAME):") dbLoadRecords("../../db/somerecords.db","P=$(IOCSH_NAME):") +dbLoadRecords("../../db/archive.db", "P=$(IOCSH_NAME):") iocInit() diff --git a/client/iocBoot/iocdemo/st_bugtest.cmd b/client/iocBoot/iocdemo/st_bugtest.cmd new file mode 100644 index 00000000..c1e16157 --- /dev/null +++ b/client/iocBoot/iocdemo/st_bugtest.cmd @@ -0,0 +1,31 @@ +#!../../bin/linux-x86_64-debug/demo + +## You may have to change demo to something else +## everywhere it appears in this file + +< envPaths + +## Register all support components +dbLoadDatabase("../../dbd/demo.dbd",0,0) +demo_registerRecordDeviceDriver(pdbbase) + +var(reccastTimeout, 5.0) +var(reccastMaxHoldoff, 5.0) + +epicsEnvSet("IOCNAME", "$(IOCSH_NAME)") +epicsEnvSet("ENGINEER", "myself") +epicsEnvSet("LOCATION", "myplace") + +epicsEnvSet("CONTACT", "mycontact") +epicsEnvSet("BUILDING", "mybuilding") +epicsEnvSet("SECTOR", "mysector") + +addReccasterEnvVars("CONTACT", "SECTOR") +addReccasterEnvVars("BUILDING") + +## Load record instances +dbLoadRecords("../../db/reccaster.db", "P=$(IOCSH_NAME):") +dbLoadRecords("../../db/somerecords.db","P=$(IOCSH_NAME):") +dbLoadRecords("../../db/archive_bugtest.db", "P=$(IOCSH_NAME):") + +iocInit() diff --git a/server/test-compose.yml b/server/test-compose.yml index 0a7b264e..a4d67565 100644 --- a/server/test-compose.yml +++ b/server/test-compose.yml @@ -1,3 +1,4 @@ +name: recceiver-test-compose services: cf: image: ghcr.io/channelfinder/channelfinderservice:master @@ -42,8 +43,6 @@ services: interval: 10s timeout: 60s retries: 5 - volumes: - - channelfinder-es-data:/usr/share/elasticsearch/data recc1: build: . @@ -143,10 +142,6 @@ services: condition: service_healthy restart: true -volumes: - channelfinder-es-data: - driver: local - networks: net-2-cf: driver: bridge diff --git a/server/test-remove-compose.yml b/server/test-remove-compose.yml new file mode 100644 index 00000000..3efa0363 --- /dev/null +++ b/server/test-remove-compose.yml @@ -0,0 +1,88 @@ +name: recceiver-test-remove-compose +services: + cf: + image: ghcr.io/channelfinder/channelfinderservice:master + hostname: cf + networks: + - net-2-cf + ports: + - "8080:8080" + depends_on: + elasticsearch: + condition: service_healthy + restart: true + environment: + ELASTICSEARCH_NETWORK_HOST: elasticsearch-cf + ELASTICSEARCH_QUERY_SIZE: 10000 + demo_auth.enabled: true + demo_auth.users: admin + demo_auth.pwds: password + demo_auth.roles: ADMIN + EPICS_PVAS_INTF_ADDR_LIST: "0.0.0.0" + aa.enabled: false + logging.level.org.springframework.web: INFO + healthcheck: + test: curl -s -f http://cf:8080/ChannelFinder + interval: 10s + timeout: 60s + retries: 5 + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4 + hostname: elasticsearch-cf + networks: + - net-2-cf + environment: + cluster.name: channelfinder + discovery.type: single-node + bootstrap.memory_lock: "true" + xpack.security.enabled: "false" + logger.level: WARN + healthcheck: + test: curl -s -f http://localhost:9200/_cluster/health + interval: 10s + timeout: 60s + retries: 5 + + recc1: + build: . + hostname: recc1 + networks: + - net-0-recc-1 + - net-2-cf + depends_on: + cf: + condition: service_healthy + restart: true + healthcheck: + test: netstat | grep cf + interval: 10s + timeout: 30s + retries: 3 + volumes: + - type: bind + source: docker/config/cf1.conf + target: /home/recceiver/cf.conf + read_only: true + - type: bind + source: docker/config/cf1.conf + target: /home/recceiver/channelfinderapi.conf + read_only: true + ioc1-1: + build: ../client + hostname: ioc1-1 + command: bash + tty: true + environment: + - IOCSH_NAME=IOC1-1 + networks: + - net-0-recc-1 + depends_on: + recc1: + condition: service_healthy + restart: true +networks: + net-2-cf: + driver: bridge + net-0-recc-1: + driver: bridge diff --git a/server/tests/__init__.py b/server/tests/__init__.py index e69de29b..795a66e9 100644 --- a/server/tests/__init__.py +++ b/server/tests/__init__.py @@ -0,0 +1,7 @@ +import logging + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + encoding="utf-8", +) diff --git a/server/tests/client_checks.py b/server/tests/client_checks.py new file mode 100644 index 00000000..cedf5465 --- /dev/null +++ b/server/tests/client_checks.py @@ -0,0 +1,71 @@ +import logging +import time +from typing import Callable + +from channelfinder import ChannelFinderClient +from testcontainers.compose import DockerCompose + +LOG: logging.Logger = logging.getLogger(__name__) + +ACTIVE_PROPERTY = {"name": "pvStatus", "owner": "admin", "value": "Active", "channels": []} +INACTIVE_PROPERTY = {"name": "pvStatus", "owner": "admin", "value": "Inactive", "channels": []} +MAX_WAIT_SECONDS = 180 +TIME_PERIOD_INCREMENT = 2 + + +def channel_match(channel0, channel1, properties_to_match: list[str]): + assert channel0["name"] == channel1["name"] + assert channel0["owner"] == channel1["owner"] + + for prop in channel0["properties"]: + assert not (prop["name"] in properties_to_match and prop not in channel1["properties"]), ( + f"Property {prop} not found in channel {channel1['name']}" + ) + assert True + + +def channels_match(channels_begin, channels_end, properties_to_match: list[str]): + for channel_index, channel in enumerate(channels_begin): + channel_match(channel, channels_end[channel_index], properties_to_match) + + +def check_channel_count(cf_client: ChannelFinderClient, name="*", expected_channel_count=24): + channels = cf_client.find(name=name) + LOG.debug("Found %s channels", len(channels)) + return len(channels) == expected_channel_count + + +def check_channel_property(cf_client: ChannelFinderClient, name="*", prop=ACTIVE_PROPERTY): + channels = cf_client.find(name=name) + active_channels = (prop in channel["properties"] for channel in channels) + return all(active_channels) + + +def wait_for_sync(cf_client: ChannelFinderClient, check: Callable[[ChannelFinderClient], bool]) -> bool: + time_period_to_wait_seconds = 1 + total_seconds_waited = 0 + while total_seconds_waited < MAX_WAIT_SECONDS: + if check(cf_client): + return True + time.sleep(time_period_to_wait_seconds) + total_seconds_waited += time_period_to_wait_seconds + time_period_to_wait_seconds += TIME_PERIOD_INCREMENT + return False + + +def create_client_and_wait(compose: DockerCompose, expected_channel_count=24) -> ChannelFinderClient: + LOG.info("Waiting for channels to sync") + cf_client = create_client_from_compose(compose) + assert wait_for_sync( + cf_client, lambda cf_client: check_channel_count(cf_client, expected_channel_count=expected_channel_count) + ) + return cf_client + + +def create_client_from_compose(compose: DockerCompose) -> ChannelFinderClient: + cf_host, cf_port = compose.get_service_host_and_port("cf") + cf_url = f"http://{cf_host if cf_host else 'localhost'}:{cf_port}/ChannelFinder" + # wait for channels to sync + LOG.info("CF URL: %s", cf_url) + cf_client = ChannelFinderClient(BaseURL=cf_url, username="admin", password="password") + return cf_client diff --git a/server/tests/docker.py b/server/tests/docker.py index 35fe2940..9afcc195 100644 --- a/server/tests/docker.py +++ b/server/tests/docker.py @@ -9,12 +9,12 @@ LOG: logging.Logger = logging.getLogger(__name__) -def test_compose() -> DockerCompose: +def test_compose(compose_file="test-compose.yml") -> DockerCompose: current_path = pathlib.Path(__file__).parent.resolve() return DockerCompose( str(current_path.parent.resolve()), - compose_file_name=str(current_path.parent.joinpath("test-compose.yml").resolve()), + compose_file_name=str(current_path.parent.joinpath(compose_file).resolve()), build=True, ) @@ -40,3 +40,10 @@ def setup_compose(): if LOG.level <= logging.DEBUG: fetch_containers_and_log_logs(compose) compose.stop() + + +def restart_container(compose: DockerCompose, host_name: str) -> None: + container = compose.get_container(host_name) + docker_client = DockerClient() + docker_client.containers.get(container.ID).stop() + docker_client.containers.get(container.ID).start() diff --git a/server/tests/test_e2e.py b/server/tests/test_e2e.py index 66bdb68a..3ba10435 100644 --- a/server/tests/test_e2e.py +++ b/server/tests/test_e2e.py @@ -1,73 +1,32 @@ import logging -import time import pytest from channelfinder import ChannelFinderClient +from .client_checks import create_client_and_wait from .docker import setup_compose # noqa: F401 -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - encoding="utf-8", -) - LOG: logging.Logger = logging.getLogger(__name__) -MAX_WAIT_SECONDS = 180 -TIME_PERIOD_INCREMENT = 2 - +RECSYNC_RESTART_DELAY = 30 # Number of channels expected in the default setup # 4 iocs, 6 channels per ioc (2 reccaster.db, 2 somerecords.db, 2 aliases in somerecords.db) -EXPECTED_DEFAULT_CHANNELS = 24 - - -def check_channel_count(cf_client, name="*"): - channels = cf_client.find(name=name) - LOG.debug("Found %s channels", len(channels)) - return len(channels) == EXPECTED_DEFAULT_CHANNELS - - -def wait_for_sync(cf_client, check): - time_period_to_wait_seconds = 1 - total_seconds_waited = 0 - while total_seconds_waited < MAX_WAIT_SECONDS: - if check(cf_client): - break - time.sleep(time_period_to_wait_seconds) - total_seconds_waited += time_period_to_wait_seconds - time_period_to_wait_seconds += TIME_PERIOD_INCREMENT - - -def create_client_and_wait(compose): - LOG.info("Waiting for channels to sync") - cf_client = create_client_from_compose(compose) - wait_for_sync(cf_client, lambda cf_client: check_channel_count(cf_client)) - return cf_client - - -def create_client_from_compose(compose): - cf_host, cf_port = compose.get_service_host_and_port("cf") - cf_url = f"http://{cf_host if cf_host else 'localhost'}:{cf_port}/ChannelFinder" - # wait for channels to sync - LOG.info("CF URL: %s", cf_url) - cf_client = ChannelFinderClient(BaseURL=cf_url) - return cf_client +EXPECTED_DEFAULT_CHANNEL_COUNT = 32 @pytest.fixture(scope="class") def cf_client(setup_compose): # noqa: F811 - return create_client_and_wait(setup_compose) + return create_client_and_wait(setup_compose, EXPECTED_DEFAULT_CHANNEL_COUNT) class TestE2E: - def test_number_of_channels_and_channel_name(self, cf_client) -> None: + def test_number_of_channels_and_channel_name(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(name="*") - assert len(channels) == EXPECTED_DEFAULT_CHANNELS + assert len(channels) == EXPECTED_DEFAULT_CHANNEL_COUNT assert channels[0]["name"] == "IOC1-1:Msg-I" # Smoke Test Default Properties - def test_number_of_aliases_and_alais_property(self, cf_client) -> None: + def test_number_of_aliases_and_alais_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("alias", "*")]) assert len(channels) == 8 assert channels[0]["name"] == "IOC1-1:lix1" @@ -78,7 +37,7 @@ def test_number_of_aliases_and_alais_property(self, cf_client) -> None: "channels": [], } in channels[0]["properties"] - def test_number_of_recordDesc_and_property(self, cf_client) -> None: + def test_number_of_recordDesc_and_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("recordDesc", "*")]) assert len(channels) == 4 assert { @@ -88,9 +47,9 @@ def test_number_of_recordDesc_and_property(self, cf_client) -> None: "channels": [], } in channels[0]["properties"] - def test_number_of_recordType_and_property(self, cf_client) -> None: + def test_number_of_recordType_and_property(self, cf_client: ChannelFinderClient) -> None: channels = cf_client.find(property=[("recordType", "*")]) - assert len(channels) == EXPECTED_DEFAULT_CHANNELS + assert len(channels) == EXPECTED_DEFAULT_CHANNEL_COUNT assert { "name": "recordType", "value": "stringin", diff --git a/server/tests/test_remove_property.py b/server/tests/test_remove_property.py new file mode 100644 index 00000000..6c604a5c --- /dev/null +++ b/server/tests/test_remove_property.py @@ -0,0 +1,93 @@ +import logging +import threading + +import pytest +from testcontainers.compose import DockerCompose + +from docker import DockerClient +from docker.models.containers import Container + +from .client_checks import INACTIVE_PROPERTY, check_channel_property, create_client_and_wait, wait_for_sync +from .docker import fetch_containers_and_log_logs, test_compose # noqa: F401 + +LOG: logging.Logger = logging.getLogger(__name__) + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + encoding="utf-8", +) + + +@pytest.fixture(scope="class") +def setup_compose(): + LOG.info("Setup remove test environment") + compose = test_compose("test-remove-compose.yml") + compose.start() + yield compose + LOG.info("Teardown test environment") + LOG.info("Stopping docker compose") + if LOG.level <= logging.DEBUG: + fetch_containers_and_log_logs(compose) + compose.stop() + + +def docker_exec_new_command(container: Container, command: str): + def stream_logs(exec_result, cmd: str): + if LOG.level <= logging.DEBUG: + LOG.debug("Logs from %s with command %s", container.name, cmd) + for line in exec_result.output: + LOG.debug(line.decode().strip()) + + exec_result = container.exec_run(command, tty=True, stream=True) + log_thread = threading.Thread( + target=stream_logs, + args=( + exec_result, + command, + ), + ) + log_thread.start() + + +class TestRemoveProperty: + def test_remove_property(self, setup_compose: DockerCompose) -> None: # noqa: F811 + """ + Test that the setup in the docker compose creates channels in channelfinder + """ + ioc_container = setup_compose.get_container("ioc1-1") + docker_client = DockerClient() + docker_ioc = docker_client.containers.get(ioc_container.ID) + docker_exec_new_command(docker_ioc, "./demo /recsync/iocBoot/iocdemo/st.cmd") + + LOG.info("Waiting for channels to sync") + cf_client = create_client_and_wait(setup_compose, expected_channel_count=8) + + # Check ioc1-1 has ai:archive with info tag "archive" + LOG.debug('Checking ioc1-1 has ai:archive with info tag "archive"') + archive_channel_name = "IOC1-1:ai:archive" + archive_channel = cf_client.find(name=archive_channel_name) + + def get_len_archive_properties(archive_channel): + return len([prop for prop in archive_channel[0]["properties"] if prop["name"] == "archive"]) + + assert get_len_archive_properties(archive_channel) == 1 + + docker_ioc.stop() + LOG.info("Waiting for channels to go inactive") + assert wait_for_sync( + cf_client, + lambda cf_client: check_channel_property(cf_client, name=archive_channel_name, prop=INACTIVE_PROPERTY), + ) + docker_ioc.start() + + docker_exec_new_command(docker_ioc, "./demo /recsync/iocBoot/iocdemo/st_bugtest.cmd") + # Detach by not waiting for the thread to finish + + LOG.debug("ioc1-1 restart") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, name=archive_channel_name)) + LOG.debug("ioc1-1 has restarted and synced") + + archive_channel = cf_client.find(name=archive_channel_name) + LOG.debug("archive channel: %s", archive_channel) + assert get_len_archive_properties(archive_channel) == 0 diff --git a/server/tests/test_restart.py b/server/tests/test_restart.py new file mode 100644 index 00000000..b7fbc4b1 --- /dev/null +++ b/server/tests/test_restart.py @@ -0,0 +1,47 @@ +import logging + +import pytest +from channelfinder import ChannelFinderClient +from testcontainers.compose import DockerCompose + +from .client_checks import channels_match, check_channel_property, create_client_and_wait, wait_for_sync +from .docker import restart_container, setup_compose # noqa: F401 + +PROPERTIES_TO_MATCH = ["pvStatus", "recordType", "recordDesc", "alias", "hostName", "iocName", "recceiverID"] + +LOG: logging.Logger = logging.getLogger(__name__) + +EXPECTED_DEFAULT_CHANNEL_COUNT = 32 + + +@pytest.fixture(scope="class") +def cf_client(setup_compose): # noqa: F811 + return create_client_and_wait(setup_compose, expected_channel_count=EXPECTED_DEFAULT_CHANNEL_COUNT) + + +class TestRestartIOC: + def test_channels_same_after_restart(self, setup_compose: DockerCompose, cf_client: ChannelFinderClient) -> None: # noqa: F811 + channels_begin = cf_client.find(name="*") + restart_container(setup_compose, "ioc1-1") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, "IOC1-1:Msg-I")) + channels_end = cf_client.find(name="*") + assert len(channels_begin) == len(channels_end) + channels_match(channels_begin, channels_end, PROPERTIES_TO_MATCH) + + def test_manual_channels_same_after_restart( + self, + setup_compose: DockerCompose, # noqa: F811 + cf_client: ChannelFinderClient, + ) -> None: + test_property = {"name": "test_property", "owner": "testowner"} + cf_client.set(properties=[test_property]) + test_property_value = test_property | {"value": "test_value"} + channels = cf_client.find(name="IOC1-1:Msg-I") + channels[0]["properties"] = [test_property_value] + cf_client.set(property=test_property) + channels_begin = cf_client.find(name="*") + restart_container(setup_compose, "ioc1-1") + assert wait_for_sync(cf_client, lambda cf_client: check_channel_property(cf_client, "IOC1-1:Msg-I")) + channels_end = cf_client.find(name="*") + assert len(channels_begin) == len(channels_end) + channels_match(channels_begin, channels_end, PROPERTIES_TO_MATCH + ["test_property"]) From 78c1022d67cea90b8f1c284367699daa162fa0be Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 28 Mar 2025 15:04:36 +0100 Subject: [PATCH 06/10] consolidate compose files --- .../{example-compose.yml => ioc-compose.yml} | 6 +- server/compose.yml | 30 ++++ .../cf-compose.yml} | 41 +---- server/docker/config/cf.conf | 5 + server/docker/config/cf1.conf | 4 +- server/docker/config/cf2.conf | 4 +- server/docker/test-multi-recc.yml | 74 +++++++++ server/docker/test-remove-infotag.yml | 34 ++++ server/test-compose.yml | 151 ------------------ server/tests/docker.py | 6 +- server/tests/test_remove_property.py | 3 +- 11 files changed, 154 insertions(+), 204 deletions(-) rename client/{example-compose.yml => ioc-compose.yml} (65%) create mode 100644 server/compose.yml rename server/{test-remove-compose.yml => docker/cf-compose.yml} (58%) create mode 100644 server/docker/config/cf.conf create mode 100644 server/docker/test-multi-recc.yml create mode 100644 server/docker/test-remove-infotag.yml delete mode 100644 server/test-compose.yml diff --git a/client/example-compose.yml b/client/ioc-compose.yml similarity index 65% rename from client/example-compose.yml rename to client/ioc-compose.yml index 7b7ca6e6..881c4f56 100644 --- a/client/example-compose.yml +++ b/client/ioc-compose.yml @@ -3,11 +3,11 @@ services: ioc1: build: ../client environment: - - IOCSH_NAME=IOC1-2 + - IOCSH_NAME=IOC1 tty: true networks: - - net-recc-1 + - net-1-recc-1 networks: - net-recc-1: + net-1-recc-1: driver: bridge diff --git a/server/compose.yml b/server/compose.yml new file mode 100644 index 00000000..af5acc34 --- /dev/null +++ b/server/compose.yml @@ -0,0 +1,30 @@ +include: + - docker/cf-compose.yml +services: + recc: + build: . + depends_on: + cf: + condition: service_healthy + restart: true + healthcheck: + test: netstat | grep cf + interval: 10s + timeout: 30s + retries: 3 + volumes: + - type: bind + source: docker/config/cf1.conf + target: /home/recceiver/cf.conf + read_only: true + - type: bind + source: docker/config/cf.conf + target: /home/recceiver/channelfinderapi.conf + read_only: true + networks: + - net-1-recc-1 + - net-2-cf + +networks: + net-1-recc-1: + driver: bridge diff --git a/server/test-remove-compose.yml b/server/docker/cf-compose.yml similarity index 58% rename from server/test-remove-compose.yml rename to server/docker/cf-compose.yml index 3efa0363..c14ac37f 100644 --- a/server/test-remove-compose.yml +++ b/server/docker/cf-compose.yml @@ -1,4 +1,4 @@ -name: recceiver-test-remove-compose +name: cf-compose services: cf: image: ghcr.io/channelfinder/channelfinderservice:master @@ -44,45 +44,6 @@ services: timeout: 60s retries: 5 - recc1: - build: . - hostname: recc1 - networks: - - net-0-recc-1 - - net-2-cf - depends_on: - cf: - condition: service_healthy - restart: true - healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 - volumes: - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/cf.conf - read_only: true - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/channelfinderapi.conf - read_only: true - ioc1-1: - build: ../client - hostname: ioc1-1 - command: bash - tty: true - environment: - - IOCSH_NAME=IOC1-1 - networks: - - net-0-recc-1 - depends_on: - recc1: - condition: service_healthy - restart: true networks: net-2-cf: driver: bridge - net-0-recc-1: - driver: bridge diff --git a/server/docker/config/cf.conf b/server/docker/config/cf.conf new file mode 100644 index 00000000..b7c239e4 --- /dev/null +++ b/server/docker/config/cf.conf @@ -0,0 +1,5 @@ + +[DEFAULT] +BaseURL = http://cf:8080/ChannelFinder +username = admin +password = password diff --git a/server/docker/config/cf1.conf b/server/docker/config/cf1.conf index c62a412f..b1ac4482 100644 --- a/server/docker/config/cf1.conf +++ b/server/docker/config/cf1.conf @@ -29,7 +29,5 @@ recordDesc = on # Mark all channels as 'Inactive' when processor is started (default: True) cleanOnStart = True -[DEFAULT] -BaseURL = http://cf:8080/ChannelFinder +# Name to used as owner username = admin -password = password diff --git a/server/docker/config/cf2.conf b/server/docker/config/cf2.conf index 635df032..18664199 100644 --- a/server/docker/config/cf2.conf +++ b/server/docker/config/cf2.conf @@ -29,7 +29,5 @@ recordDesc = on # Mark all channels as 'Inactive' when processor is started (default: True) cleanOnStart = True -[DEFAULT] -BaseURL = http://cf:8080/ChannelFinder +# Name to used as owner username = admin -password = password diff --git a/server/docker/test-multi-recc.yml b/server/docker/test-multi-recc.yml new file mode 100644 index 00000000..f66fa173 --- /dev/null +++ b/server/docker/test-multi-recc.yml @@ -0,0 +1,74 @@ +name: test-multi-recc +include: + - cf-compose.yml +services: + recc1: + extends: + file: ../compose.yml + service: recc + depends_on: + cf: + condition: service_healthy + restart: true + hostname: recc1 + networks: + - net-1-recc-1 + - net-2-cf + ioc1-1: + extends: + file: ../../client/ioc-compose.yml + service: ioc1 + depends_on: + recc1: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC1-1 + hostname: ioc1-1 + networks: + - net-1-recc-1 + ioc1-2: + extends: + ioc1-1 + depends_on: + recc1: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC1-2 + hostname: ioc1-1 + recc2: + extends: + recc1 + hostname: recc2 + networks: + - net-1-recc-2 + - net-2-cf + ioc2-1: + extends: + ioc1-1 + depends_on: + recc2: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC2-1 + hostname: ioc2-1 + networks: + - net-1-recc-2 + ioc2-2: + extends: + ioc2-1 + depends_on: + recc2: + condition: service_healthy + restart: true + environment: + - IOCSH_NAME=IOC2-2 + hostname: ioc2-2 + +networks: + net-1-recc-1: + driver: bridge + net-1-recc-2: + driver: bridge diff --git a/server/docker/test-remove-infotag.yml b/server/docker/test-remove-infotag.yml new file mode 100644 index 00000000..ae4a43aa --- /dev/null +++ b/server/docker/test-remove-infotag.yml @@ -0,0 +1,34 @@ +name: test-removeinfotag +include: + - cf-compose.yml +services: + recc1: + extends: + file: ../compose.yml + service: recc + depends_on: + cf: + condition: service_healthy + restart: true + hostname: recc1 + networks: + - net-1-recc-1 + - net-2-cf + ioc1-1: + extends: + file: ../../client/ioc-compose.yml + service: ioc1 + environment: + - IOCSH_NAME=IOC1-1 + depends_on: + recc1: + condition: service_healthy + restart: true + hostname: ioc1-1 + command: bash + networks: + - net-1-recc-1 + +networks: + net-1-recc-1: + driver: bridge diff --git a/server/test-compose.yml b/server/test-compose.yml deleted file mode 100644 index a4d67565..00000000 --- a/server/test-compose.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: recceiver-test-compose -services: - cf: - image: ghcr.io/channelfinder/channelfinderservice:master - hostname: cf - networks: - - net-2-cf - ports: - - "8080:8080" - depends_on: - elasticsearch: - condition: service_healthy - restart: true - environment: - ELASTICSEARCH_NETWORK_HOST: elasticsearch-cf - ELASTICSEARCH_QUERY_SIZE: 10000 - demo_auth.enabled: true - demo_auth.users: admin - demo_auth.pwds: password - demo_auth.roles: ADMIN - EPICS_PVAS_INTF_ADDR_LIST: "0.0.0.0" - aa.enabled: false - logging.level.org.springframework.web: INFO - healthcheck: - test: curl -s -f http://cf:8080/ChannelFinder - interval: 10s - timeout: 60s - retries: 5 - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4 - hostname: elasticsearch-cf - networks: - - net-2-cf - environment: - cluster.name: channelfinder - discovery.type: single-node - bootstrap.memory_lock: "true" - xpack.security.enabled: "false" - logger.level: WARN - healthcheck: - test: curl -s -f http://localhost:9200/_cluster/health - interval: 10s - timeout: 60s - retries: 5 - - recc1: - build: . - hostname: recc1 - networks: - - net-0-recc-1 - - net-2-cf - depends_on: - cf: - condition: service_healthy - restart: true - healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 - volumes: - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/cf.conf - read_only: true - - type: bind - source: docker/config/cf1.conf - target: /home/recceiver/channelfinderapi.conf - read_only: true - recc2: - build: . - hostname: recc2 - networks: - - net-1-recc-2 - - net-2-cf - depends_on: - cf: - condition: service_healthy - restart: true - healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 - volumes: - - type: bind - source: docker/config/cf2.conf - target: /home/recceiver/cf.conf - read_only: true - - type: bind - source: docker/config/cf2.conf - target: /home/recceiver/channelfinderapi.conf - read_only: true - - ioc1-1: - build: ../client - hostname: ioc1-1 - tty: true - environment: - - IOCSH_NAME=IOC1-1 - networks: - - net-0-recc-1 - depends_on: - recc1: - condition: service_healthy - restart: true - ioc1-2: - build: ../client - hostname: ioc1-2 - tty: true - environment: - - IOCSH_NAME=IOC1-2 - networks: - - net-0-recc-1 - depends_on: - recc2: - condition: service_healthy - restart: true - ioc2-1: - build: ../client - hostname: ioc2-1 - tty: true - environment: - - IOCSH_NAME=IOC2-1 - networks: - - net-1-recc-2 - depends_on: - cf: - condition: service_healthy - restart: true - ioc2-2: - build: ../client - hostname: ioc2-2 - tty: true - environment: - - IOCSH_NAME=IOC2-2 - networks: - - net-1-recc-2 - depends_on: - cf: - condition: service_healthy - restart: true - -networks: - net-2-cf: - driver: bridge - net-0-recc-1: - driver: bridge - net-1-recc-2: - driver: bridge diff --git a/server/tests/docker.py b/server/tests/docker.py index 9afcc195..455cf7a9 100644 --- a/server/tests/docker.py +++ b/server/tests/docker.py @@ -1,5 +1,5 @@ import logging -import pathlib +from pathlib import Path import pytest from testcontainers.compose import DockerCompose @@ -9,8 +9,8 @@ LOG: logging.Logger = logging.getLogger(__name__) -def test_compose(compose_file="test-compose.yml") -> DockerCompose: - current_path = pathlib.Path(__file__).parent.resolve() +def test_compose(compose_file=Path("docker") / Path("test-multi-recc.yml")) -> DockerCompose: + current_path = Path(__file__).parent.resolve() return DockerCompose( str(current_path.parent.resolve()), diff --git a/server/tests/test_remove_property.py b/server/tests/test_remove_property.py index 6c604a5c..7b5950d3 100644 --- a/server/tests/test_remove_property.py +++ b/server/tests/test_remove_property.py @@ -1,5 +1,6 @@ import logging import threading +from pathlib import Path import pytest from testcontainers.compose import DockerCompose @@ -22,7 +23,7 @@ @pytest.fixture(scope="class") def setup_compose(): LOG.info("Setup remove test environment") - compose = test_compose("test-remove-compose.yml") + compose = test_compose(Path("docker") / Path("test-remove-infotag.yml")) compose.start() yield compose LOG.info("Teardown test environment") From 605ba5d25fa0f6a87f90080d671d19a97bef1afc Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 28 Mar 2025 15:12:12 +0100 Subject: [PATCH 07/10] Add repr to transaction for debugging --- server/recceiver/recast.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/recceiver/recast.py b/server/recceiver/recast.py index 3c0c6e30..b0abf6b4 100644 --- a/server/recceiver/recast.py +++ b/server/recceiver/recast.py @@ -241,6 +241,18 @@ def __str__(self): source_address, init, conn, nenv, nadd, nalias, ninfo, ndel ) + def __repr__(self): + return f"""Transaction( + source_address={self.source_address}, + initial={self.initial}, + connected={self.connected}, + records_to_add={self.records_to_add}, + client_infos={self.client_infos}, + record_infos_to_add={self.record_infos_to_add}, + aliases={self.aliases}, + records_to_delete={self.records_to_delete}) + """ + class CollectionSession(object): timeout = 5.0 From fdb880fb33c1eaa19e6e71d89296d6167cf3b12d Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 28 Mar 2025 15:20:28 +0100 Subject: [PATCH 08/10] Add some more debugs to cfstore To find source of problem --- server/recceiver/cfstore.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index c31e867f..ba9107bc 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -192,6 +192,7 @@ def _commitWithThread(self, transaction): ) _log.info("CF_COMMIT: {transaction}".format(transaction=transaction)) + _log.debug("CF_COMMIT: transaction: {s}".format(s=repr(transaction))) """ a dictionary with a list of records with their associated property info pvInfo @@ -447,7 +448,11 @@ def __updateCF__( iocTime, ): _log.info("CF Update IOC: {iocid}".format(iocid=iocid)) - + _log.debug( + "CF Update IOC: {iocid} recordInfoByName {recordInfoByName}".format( + iocid=iocid, recordInfoByName=recordInfoByName + ) + ) # Consider making this function a class methed then 'processor' simply becomes 'self' client = processor.client channels_dict = processor.channel_dict @@ -483,6 +488,7 @@ def __updateCF__( if ( len(new_channels) == 0 or cf_channel["name"] in records_to_delete ): # case: empty commit/del, remove all reference to ioc + _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel["name"])) if cf_channel["name"] in channels_dict: cf_channel["owner"] = iocs[channels_dict[cf_channel["name"]][-1]]["owner"] cf_channel["properties"] = __merge_property_lists( @@ -560,6 +566,11 @@ def __updateCF__( Channel exists in Channelfinder with same hostname and iocname. Update the status to ensure it is marked active and update the time. """ + _log.debug( + "Channel {s} exists in Channelfinder with same hostname and iocname".format( + s=cf_channel["name"] + ) + ) cf_channel["properties"] = __merge_property_lists( [ create_active_property(owner), @@ -644,7 +655,10 @@ def __updateCF__( newProps = newProps + recordInfoByName[channel_name]["infoProperties"] if channel_name in existingChannels: - """update existing channel: exists but with a different hostName and/or iocName""" + _log.debug( + f"""update existing channel{channel_name}: exists but with a different hostName and/or iocName""" + ) + existingChannel = existingChannels[channel_name] existingChannel["properties"] = __merge_property_lists(newProps, existingChannel) channels.append(existingChannel) From e72c2f3e64c739924d2df203f4d73700c8088522 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 20 Mar 2025 15:50:30 +0100 Subject: [PATCH 09/10] Fixes the problem Essentially when merging property lists, should only take the old data when setting the channel to be inactive. Otherwise only the new data owned by the recceiver is relevant --- server/recceiver/cfstore.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ba9107bc..115a4e7c 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -124,6 +124,7 @@ def _startServiceWithLock(self): self.client.set(property={"name": cf_property, "owner": owner}) self.record_property_names_list = set(record_property_names_list) + self.managed_properties = required_properties.union(record_property_names_list) _log.debug("record_property_names_list = {}".format(self.record_property_names_list)) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") @@ -213,6 +214,7 @@ def _commitWithThread(self, transaction): """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) + _log.debug("transaction: {s}".format(s=repr(transaction))) recordInfo = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): @@ -274,8 +276,6 @@ def _commitWithThread(self, transaction): ) continue recordInfoByName[info["pvName"]] = info - _log.debug("Add record: {record_id}: {info}".format(record_id=record_id, info=info)) - _log.debug("Add record: {record_id}: {info}".format(record_id=record_id, info=info)) if transaction.initial: """Add IOC to source list """ @@ -494,6 +494,7 @@ def __updateCF__( cf_channel["properties"] = __merge_property_lists( create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), cf_channel, + processor.managed_properties, ) if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( @@ -503,6 +504,7 @@ def __updateCF__( ) ), cf_channel, + processor.managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) @@ -522,6 +524,7 @@ def __updateCF__( cf_channel, ), alias, + processor.managed_properties, ) if conf.get("recordType"): cf_channel["properties"] = __merge_property_lists( @@ -532,6 +535,7 @@ def __updateCF__( ) ), cf_channel, + processor.managed_properties, ) channels.append(alias) _log.debug("Add existing alias to previous IOC: {s}".format(s=channels[-1])) @@ -577,6 +581,7 @@ def __updateCF__( create_time_property(owner, iocTime), ], cf_channel, + processor.managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel with same IOC: {s}".format(s=channels[-1])) @@ -594,6 +599,7 @@ def __updateCF__( create_time_property(owner, iocTime), ], alias, + processor.managed_properties, ) channels.append(alias) new_channels.remove(alias["name"]) @@ -609,6 +615,7 @@ def __updateCF__( ), ], cf_channel, + processor.managed_properties, ) channels.append( create_channel( @@ -660,7 +667,11 @@ def __updateCF__( ) existingChannel = existingChannels[channel_name] - existingChannel["properties"] = __merge_property_lists(newProps, existingChannel) + existingChannel["properties"] = __merge_property_lists( + newProps, + existingChannel, + processor.managed_properties, + ) channels.append(existingChannel) _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" @@ -672,7 +683,11 @@ def __updateCF__( for alias in recordInfoByName[channel_name]["aliases"]: if alias in existingChannels: ach = existingChannels[alias] - ach["properties"] = __merge_property_lists(alProps, ach) + ach["properties"] = __merge_property_lists( + alProps, + ach, + processor.managed_properties, + ) channels.append(ach) else: channels.append(create_channel(alias, owner, alProps)) @@ -724,7 +739,9 @@ def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, ) -def __merge_property_lists(newProperties: list[dict[str, str]], channel: dict[str, list[dict[str, str]]]): +def __merge_property_lists( + newProperties: list[dict[str, str]], channel: dict[str, list[dict[str, str]]], managed_properties=set() +) -> list[dict[str, str]]: """ Merges two lists of properties ensuring that there are no 2 properties with the same name In case of overlap between the new and old property lists the @@ -732,7 +749,7 @@ def __merge_property_lists(newProperties: list[dict[str, str]], channel: dict[st """ newPropNames = [p["name"] for p in newProperties] for oldProperty in channel["properties"]: - if oldProperty["name"] not in newPropNames: + if oldProperty["name"] not in newPropNames and (oldProperty["name"] not in managed_properties): newProperties = newProperties + [oldProperty] return newProperties From d17610d3ad2b5a63fdf849b28b919421830b82df Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 1 Apr 2025 10:28:37 +0200 Subject: [PATCH 10/10] Remove unneeded assert True --- server/tests/client_checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/tests/client_checks.py b/server/tests/client_checks.py index cedf5465..9af7ce0e 100644 --- a/server/tests/client_checks.py +++ b/server/tests/client_checks.py @@ -21,7 +21,6 @@ def channel_match(channel0, channel1, properties_to_match: list[str]): assert not (prop["name"] in properties_to_match and prop not in channel1["properties"]), ( f"Property {prop} not found in channel {channel1['name']}" ) - assert True def channels_match(channels_begin, channels_end, properties_to_match: list[str]):