From 9aec778b424e31e7ae096e3c96a575a8a5654eb9 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 8 Dec 2023 09:23:47 +0100 Subject: [PATCH] Update elastic search version --- docker-compose-integrationtest.yml | 2 +- docker-compose.yml | 2 +- pom.xml | 4 +- .../channelfinder/ChannelRepository.java | 25 +- .../phoebus/channelfinder/ChannelScroll.java | 20 +- .../example/PopulateService.java | 382 +++++++++--------- .../ChannelRepositorySearchIT.java | 35 +- 7 files changed, 246 insertions(+), 224 deletions(-) diff --git a/docker-compose-integrationtest.yml b/docker-compose-integrationtest.yml index 70f2443c..47a89c34 100644 --- a/docker-compose-integrationtest.yml +++ b/docker-compose-integrationtest.yml @@ -54,7 +54,7 @@ services: retries: 5 elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.2 hostname: elasticsearch-cf networks: - channelfinder-net diff --git a/docker-compose.yml b/docker-compose.yml index 725b1571..d71307a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: java -jar /channelfinder/ChannelFinder-4.7.2.jar" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.2 hostname: elasticsearch-cf networks: - channelfinder-net diff --git a/pom.xml b/pom.xml index 8fcf70d2..0a7df699 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ UTF-8 2.7.3 - 8.2.0 + 8.11.2 5.10.0 true true @@ -100,7 +100,7 @@ co.elastic.clients elasticsearch-java - 8.2.0 + ${elasticsearch.version} com.fasterxml.jackson.core diff --git a/src/main/java/org/phoebus/channelfinder/ChannelRepository.java b/src/main/java/org/phoebus/channelfinder/ChannelRepository.java index ae7ea4f1..0c4ea1c3 100644 --- a/src/main/java/org/phoebus/channelfinder/ChannelRepository.java +++ b/src/main/java/org/phoebus/channelfinder/ChannelRepository.java @@ -3,6 +3,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.FieldSort; +import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.Result; import co.elastic.clients.elasticsearch._types.SortOptions; @@ -10,6 +11,7 @@ import co.elastic.clients.elasticsearch._types.query_dsl.DisMaxQuery; import co.elastic.clients.elasticsearch._types.query_dsl.IdsQuery; import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch._types.query_dsl.WildcardQuery; import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; @@ -33,6 +35,7 @@ import org.phoebus.channelfinder.entity.Tag; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; import org.springframework.data.repository.CrudRepository; import org.springframework.http.HttpStatus; @@ -66,7 +69,7 @@ public class ChannelRepository implements CrudRepository { @Autowired @Qualifier("indexClient") ElasticsearchClient client; - + final ObjectMapper objectMapper = new ObjectMapper() .addMixIn(Tag.class, Tag.OnlyTag.class) .addMixIn(Property.class, Property.OnlyProperty.class); @@ -422,7 +425,7 @@ public SearchResult search(MultiValueMap searchParameters) { .size(finalSize) .trackTotalHits(builder -> builder.enabled(builtQuery.trackTotalHits)) .sort(SortOptions.of(o -> o.field(FieldSort.of(f -> f.field("name"))))); - builtQuery.searchAfter.ifPresent(searchBuilder::searchAfter); + builtQuery.searchAfter.ifPresent(s -> searchBuilder.searchAfter(FieldValue.of(s))); SearchResponse response = client.search(searchBuilder.build(), Channel.class @@ -448,7 +451,7 @@ private BuiltQuery getBuiltQuery(MultiValueMap searchParameters) int from = 0; boolean trackTotalHits = false; Optional searchAfter = Optional.empty(); - String valueSplitPattern = "[\\|,;]"; + String valueSplitPattern = "[|,;]"; for (Map.Entry> parameter : searchParameters.entrySet()) { String key = parameter.getKey().trim(); boolean isNot = key.endsWith("!"); @@ -460,7 +463,7 @@ private BuiltQuery getBuiltQuery(MultiValueMap searchParameters) for (String value : parameter.getValue()) { DisMaxQuery.Builder nameQuery = new DisMaxQuery.Builder(); for (String pattern : value.split(valueSplitPattern)) { - nameQuery.queries(WildcardQuery.of(w -> w.field("name").caseInsensitive(true).value(pattern.trim()))._toQuery()); + nameQuery.queries(getSingleValueQuery("name", pattern.trim())); } boolQuery.must(nameQuery.build()._toQuery()); } @@ -471,7 +474,7 @@ private BuiltQuery getBuiltQuery(MultiValueMap searchParameters) for (String pattern : value.split(valueSplitPattern)) { tagQuery.queries( NestedQuery.of(n -> n.path("tags").query( - WildcardQuery.of(w -> w.field("tags.name").caseInsensitive(true).value(pattern.trim()))._toQuery()))._toQuery()); + getSingleValueQuery("tags.name", pattern.trim())))._toQuery()); } if (isNot) { boolQuery.mustNot(tagQuery.build()._toQuery()); @@ -510,11 +513,11 @@ private BuiltQuery getBuiltQuery(MultiValueMap searchParameters) String finalKey = key; BoolQuery bq; if (isNot) { - bq = BoolQuery.of(p -> p.must(WildcardQuery.of(name -> name.field("properties.name").caseInsensitive(true).value(finalKey))._toQuery()) - .mustNot(WildcardQuery.of(val -> val.field("properties.value").caseInsensitive(true).value(pattern.trim()))._toQuery())); + bq = BoolQuery.of(p -> p.must(getSingleValueQuery("properties.name", finalKey)) + .mustNot(getSingleValueQuery("properties.value", pattern.trim()))); } else { - bq = BoolQuery.of(p -> p.must(WildcardQuery.of(name -> name.field("properties.name").caseInsensitive(true).value(finalKey))._toQuery()) - .must(WildcardQuery.of(val -> val.field("properties.value").caseInsensitive(true).value(pattern.trim()))._toQuery())); + bq = BoolQuery.of(p -> p.must(getSingleValueQuery("properties.name", finalKey)) + .must(getSingleValueQuery("properties.value", pattern.trim()))); } propertyQuery.queries( NestedQuery.of(n -> n.path("properties").query(bq._toQuery()))._toQuery() @@ -528,6 +531,10 @@ private BuiltQuery getBuiltQuery(MultiValueMap searchParameters) return new BuiltQuery(boolQuery, size, from, searchAfter, trackTotalHits); } + private static Query getSingleValueQuery(String name, String pattern) { + return WildcardQuery.of(w -> w.field(name).caseInsensitive(true).value(pattern))._toQuery(); + } + private static class BuiltQuery { public final BoolQuery.Builder boolQuery; public final Integer size; diff --git a/src/main/java/org/phoebus/channelfinder/ChannelScroll.java b/src/main/java/org/phoebus/channelfinder/ChannelScroll.java index c2903b91..4c6ad7f1 100644 --- a/src/main/java/org/phoebus/channelfinder/ChannelScroll.java +++ b/src/main/java/org/phoebus/channelfinder/ChannelScroll.java @@ -12,6 +12,7 @@ import java.util.stream.Collectors; import co.elastic.clients.elasticsearch._types.FieldSort; +import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.query_dsl.*; import co.elastic.clients.elasticsearch.core.SearchRequest; @@ -83,14 +84,15 @@ public Scroll query(@PathVariable("scrollId") String scrollId, @RequestParam Mul * The query result is sorted based on the channel name ~size - The number of * channels to be returned ~from - The starting index of the channel list * + * TODO combine with ChannelRepository code. * @param scrollId scroll ID * @param searchParameters - search parameters for scrolling searches * @return search scroll */ public Scroll search(String scrollId, MultiValueMap searchParameters) { BoolQuery.Builder boolQuery = new BoolQuery.Builder(); - Integer size = esService.getES_QUERY_SIZE(); - Integer from = 0; + int size = esService.getES_QUERY_SIZE(); + int from = 0; for (Map.Entry> parameter : searchParameters.entrySet()) { String key = parameter.getKey().trim(); @@ -127,13 +129,13 @@ public Scroll search(String scrollId, MultiValueMap searchParame case "~size": Optional maxSize = parameter.getValue().stream().max(Comparator.comparing(Integer::valueOf)); if (maxSize.isPresent()) { - size = Integer.valueOf(maxSize.get()); + size = Integer.parseInt(maxSize.get()); } break; case "~from": Optional maxFrom = parameter.getValue().stream().max(Comparator.comparing(Integer::valueOf)); if (maxFrom.isPresent()) { - from = Integer.valueOf(maxFrom.get()); + from = Integer.parseInt(maxFrom.get()); } break; @@ -161,22 +163,20 @@ public Scroll search(String scrollId, MultiValueMap searchParame } try { - Integer finalSize = size; - Integer finalFrom = from; SearchRequest.Builder builder = new SearchRequest.Builder(); builder.index(esService.getES_CHANNEL_INDEX()) .query(boolQuery.build()._toQuery()) - .from(finalFrom) - .size(finalSize) + .from(from) + .size(size) .sort(SortOptions.of(o -> o.field(FieldSort.of(f -> f.field("name"))))); if(scrollId != null && !scrollId.isEmpty()) { - builder.searchAfter(scrollId); + builder.searchAfter(FieldValue.of(scrollId)); } SearchResponse response = client.search(builder.build(), Channel.class ); List> hits = response.hits().hits(); - return new Scroll(hits.size() > 0 ? hits.get(hits.size()-1).id() : null, hits.stream().map(Hit::source).collect(Collectors.toList())); + return new Scroll(!hits.isEmpty() ? hits.get(hits.size()-1).id() : null, hits.stream().map(Hit::source).collect(Collectors.toList())); } catch (Exception e) { String message = MessageFormat.format(TextUtil.SEARCH_FAILED_CAUSE, searchParameters, e.getMessage()); logger.log(Level.SEVERE, message, e); diff --git a/src/main/java/org/phoebus/channelfinder/example/PopulateService.java b/src/main/java/org/phoebus/channelfinder/example/PopulateService.java index 00e71cf8..7f30c6a7 100644 --- a/src/main/java/org/phoebus/channelfinder/example/PopulateService.java +++ b/src/main/java/org/phoebus/channelfinder/example/PopulateService.java @@ -23,7 +23,6 @@ import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; import co.elastic.clients.elasticsearch.core.IndexRequest; -import co.elastic.clients.elasticsearch.core.IndexResponse; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; import com.fasterxml.jackson.core.type.TypeReference; @@ -67,9 +66,23 @@ public class PopulateService { private static final Logger logger = Logger.getLogger(PopulateService.class.getName()); - - static int max_prop = 40; // must be >=20 - static int max_tag = 60; // must be >=11 + public static final String DEVICE_POWER_SUPPLY = "power supply"; + public static final String DEVICE_MAGNET = "magnet"; + public static final String UNIT_TEMP = "temperature"; + public static final String SIGTYPE_STATUS = "status"; + public static final String ELEMENT_VACUUM = "vacuum"; + public static final String UNIT_FIELD = "field"; + public static final String SIGTYPE_READBACK = "readback"; + public static final String SIGTYPE_SETPOINT = "setpoint"; + public static final String UNIT_CURRENT = "current"; + public static final String UNIT_POWER = "power"; + public static final String SIGTYPE_SWITCH = "switch"; + public static final String ELEMENT_TEMPERATURE_SENSOR = "temperature sensor"; + public static final String DEVICE_SENSOR = "sensor"; + public static final String PROPERTY_NAME_MOUNT = "mount"; + + static int maxProp = 40; // must be >=20 + static int maxTag = 60; // must be >=11 // The number of cells can be increases to 100 // Each cell would consist of a 1000 SR channels and 500 BO channels @@ -79,13 +92,13 @@ public class PopulateService { static String powner = "testp"; static String towner = "testt"; - public static List val_bucket = Arrays.asList(0, 1, 2, 5, 10, 20, 50, 100, 200, 500); - public static List val_bucket_size = Arrays.asList(1000 - val_bucket.stream().mapToInt(Integer::intValue).sum(), 1, 2, 5, 10, 20, 50, 100, 200, 500); + public static final List valBucket = Arrays.asList(0, 1, 2, 5, 10, 20, 50, 100, 200, 500); + public static final List valBucketSize = Arrays.asList(1000 - valBucket.stream().mapToInt(Integer::intValue).sum(), 1, 2, 5, 10, 20, 50, 100, 200, 500); // A static list of props, tags, channels handled by this class which must be cleaned up on closure. - static Set prop_list = new HashSet<>(); - Set tag_list = new HashSet<>(); - Set channel_list = new HashSet<>(); + static Set propertySet = new HashSet<>(); + Set tagSet = new HashSet<>(); + Set channelList = new HashSet<>(); @Autowired ElasticConfig esService; @@ -95,32 +108,32 @@ public class PopulateService { ElasticsearchClient client; - public static ObjectMapper mapper = new ObjectMapper(); + public static final ObjectMapper mapper = new ObjectMapper(); static int index; - static List tokens_1000 = new ArrayList<>(); - static List tokens_500 = new ArrayList<>(); + static List tokens1000 = new ArrayList<>(); + static List tokens500 = new ArrayList<>(); // Create a list of properties static { for (int i = 10; i < 70; i++) { - prop_list.add(new Property("prop" + String.format("%03d", i), powner)); + propertySet.add(new Property("prop" + String.format("%03d", i), powner)); } index = 0; - val_bucket_size.stream().forEachOrdered(count -> { + valBucketSize.forEach(count -> { for (int i = 0; i < count; i++) { - tokens_1000.add(val_bucket.get(index)); + tokens1000.add(valBucket.get(index)); } index++; }); index = 0; - val_bucket_size.stream().forEachOrdered(count -> { + valBucketSize.forEach(count -> { for (int i = 0; i < count; i++) { - tokens_500.add(val_bucket.get(index)); + tokens500.add(valBucket.get(index)); } index++; }); @@ -128,7 +141,7 @@ public class PopulateService { public synchronized void cleanupDB() { BulkRequest.Builder br = new BulkRequest.Builder(); - for (String channelName : channel_list) { + for (String channelName : channelList) { br.operations(op -> op .delete(idx -> idx .index(esService.getES_CHANNEL_INDEX()) @@ -136,7 +149,7 @@ public synchronized void cleanupDB() { ) ); } - for (Tag tag : tag_list) { + for (Tag tag : tagSet) { br.operations(op -> op .delete(idx -> idx .index(esService.getES_TAG_INDEX()) @@ -144,7 +157,7 @@ public synchronized void cleanupDB() { ) ); } - for (Property property : prop_list) { + for (Property property : propertySet) { br.operations(op -> op .delete(idx -> idx .index(esService.getES_PROPERTY_INDEX()) @@ -173,42 +186,69 @@ public synchronized void createDB(int cells) { } public Set getChannelList() { - return channel_list; + return channelList; } public synchronized void createDB() { Collection finalResult = new ArrayList<>(); long start = System.currentTimeMillis(); + int freq = 25; + Collection channels = new ArrayList<>(); for (int i = 1; i <= numberOfCells; i++) { + logger.info("Creating SR channels for cell " + i); String cell = String.format("%03d", i); - try { - finalResult.add(insertSRCell(cell)); - } catch (Exception e) { - logger.log(Level.WARNING, e.getMessage(), e); - } + channels.addAll(insertSRCell(cell)); + if (i % freq == 0) { + try { + logger.info("Bulk inserting channels"); + + bulkInsertChannels(channels); + channels.clear(); + } catch (Exception e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + } } for (int i = 1; i <= numberOfCells; i++) { + logger.info("Creating BO channels for cell " + i); + String cell = String.format("%03d", i); - try { - finalResult.add(insertBOCell(cell)); - } catch (Exception e) { - logger.log(Level.WARNING, e.getMessage(), e); + channels.addAll(insertBOCell(cell)); + + if (i % freq == 0) { + try { + logger.info("Bulk inserting channels"); + + bulkInsertChannels(channels); + channels.clear(); + } catch (Exception e) { + logger.log(Level.WARNING, e.getMessage(), e); + } } } + try { + logger.info("Bulk inserting channels"); + + bulkInsertChannels(channels); + channels.clear(); + + } catch (Exception e) { + logger.log(Level.WARNING, e.getMessage(), e); + } final long time = System.currentTimeMillis() - start; logger.log(Level.INFO, () -> finalResult.size() + " channels created sequentially in time: " + time); - prop_list.forEach(p -> + propertySet.forEach(p -> logger.log(Level.INFO, p.toLog()) ); - tag_list.forEach(t -> + tagSet.forEach(t -> logger.log(Level.INFO, t.toLog()) ); BulkRequest.Builder br = new BulkRequest.Builder(); - for (Property property : prop_list) { + for (Property property : propertySet) { br.operations(op -> op.index(bIndex -> bIndex.index(esService.getES_PROPERTY_INDEX()).id(property.getName()).document(property))); } - for (Tag tag : tag_list) { + for (Tag tag : tagSet) { br.operations(op -> op.index(bIndex -> bIndex.index(esService.getES_TAG_INDEX()).id(tag.getName()).document(tag))); } br.refresh(Refresh.True); @@ -231,14 +271,13 @@ public synchronized void createDB() { public void createTagsAndProperties(URL tagResource, URL propertyResource) { - long start = System.currentTimeMillis(); // Setup the default tags String tagsURL; tagsURL = tagResource.toExternalForm(); try (InputStream input = new URL(tagsURL).openStream() ) { List jsonTag = mapper.readValue(input, new TypeReference>(){}); - jsonTag.stream().forEach(tag -> { + jsonTag.forEach(tag -> { try { if(!client.exists(e -> e.index(esService.getES_TAG_INDEX()).id(tag.getName())).value()){ IndexRequest indexRequest = @@ -247,7 +286,7 @@ public void createTagsAndProperties(URL tagResource, URL propertyResource) { .id(tag.getName()) .document(tag) .refresh(Refresh.True)); - IndexResponse response = client.index(indexRequest); + client.index(indexRequest); } } catch (IOException e) { logger.log(Level.WARNING, "Failed to initialize tag : " +tag.getName(), e); @@ -261,9 +300,10 @@ public void createTagsAndProperties(URL tagResource, URL propertyResource) { String propertiesURL; propertiesURL = propertyResource.toExternalForm(); try (InputStream input = new URL(propertiesURL).openStream() ) { - List jsonTag = mapper.readValue(input, new TypeReference>(){}); + List jsonTag = mapper.readValue(input, new TypeReference<>() { + }); - jsonTag.stream().forEach(property -> { + jsonTag.forEach(property -> { try { if(!client.exists(e -> e.index(esService.getES_PROPERTY_INDEX()).id(property.getName())).value()){ IndexRequest indexRequest = @@ -272,7 +312,7 @@ public void createTagsAndProperties(URL tagResource, URL propertyResource) { .id(property.getName()) .document(property) .refresh(Refresh.True)); - IndexResponse response = client.index(indexRequest); + client.index(indexRequest); } } catch (IOException e) { logger.log(Level.WARNING, "Failed to initialize property : " +property.getName(), e); @@ -282,42 +322,46 @@ public void createTagsAndProperties(URL tagResource, URL propertyResource) { logger.log(Level.WARNING, "Failed to initialize property ", ex); } } - private boolean insertSRCell(String cell) throws Exception { + private Collection insertSRCell(String cell) { String loc = "storage ring"; String pre = "SR:C"; // Tokens Map> tokens = new HashMap<>(); for (int i = 0; i < 6; i++) { - tokens.put(i, new ArrayList<>(tokens_1000)); + tokens.put(i, new ArrayList<>(tokens1000)); } AtomicInteger channelCounter = new AtomicInteger(0); Collection result = new ArrayList<>(1000); - result.addAll(insert_big_magnets(tokens, channelCounter, 2, pre, "DP", loc, cell, "dipole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 5, pre, "QDP:D", loc, cell, "defocusing quadrupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 5, pre, "QDP:F", loc, cell, "focusing quadrupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "QDP:S", loc, cell, "skew quadrupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "STP", loc, cell, "sextupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 5, pre, "HC:S", loc, cell, "horizontal slow corrector")); - result.addAll(insert_air_magnets(tokens, channelCounter, 5, pre, "HC:F", loc, cell, "horizontal fast corrector")); - result.addAll(insert_big_magnets(tokens, channelCounter, 5, pre, "VC:S", loc, cell, "vertical slow corrector")); - result.addAll(insert_air_magnets(tokens, channelCounter, 4, pre, "VC:F", loc, cell, "vertical fast corrector")); + result.addAll(insertBigMagnets(tokens, channelCounter, 2, pre, "DP", loc, cell, "dipole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 5, pre, "QDP:D", loc, cell, "defocusing quadrupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 5, pre, "QDP:F", loc, cell, "focusing quadrupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "QDP:S", loc, cell, "skew quadrupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "STP", loc, cell, "sextupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 5, pre, "HC:S", loc, cell, "horizontal slow corrector")); + result.addAll(insertAirMagnets(tokens, channelCounter, 5, pre, "HC:F", loc, cell, "horizontal fast corrector")); + result.addAll(insertBigMagnets(tokens, channelCounter, 5, pre, "VC:S", loc, cell, "vertical slow corrector")); + result.addAll(insertAirMagnets(tokens, channelCounter, 4, pre, "VC:F", loc, cell, "vertical fast corrector")); - result.addAll(insert_valves(tokens, channelCounter, 5, pre, "GV", loc, cell, "vacuum")); + result.addAll(insertValves(tokens, channelCounter, 5, pre, loc, cell)); - result.addAll(insert_gauges(tokens, channelCounter, 5, pre, "VGC", loc, cell, "vacuum")); - result.addAll(insert_gauges(tokens, channelCounter, 5, pre, "TCG", loc, cell, "vacuum")); + result.addAll(insertGauges(tokens, channelCounter, 5, pre, "VGC", loc, cell)); + result.addAll(insertGauges(tokens, channelCounter, 5, pre, "TCG", loc, cell)); - result.addAll(insert_pumps(tokens, channelCounter, 2, pre, "IPC", loc, cell, "vacuum")); - result.addAll(insert_pumps(tokens, channelCounter, 2, pre, "TMP", loc, cell, "vacuum")); + result.addAll(insertPumps(tokens, channelCounter, pre, "IPC", loc, cell)); + result.addAll(insertPumps(tokens, channelCounter, pre, "TMP", loc, cell)); - result.addAll(insert_temps(tokens, channelCounter, 40, pre, "TC", loc, cell, "temperature sensor")); + result.addAll(insertTemps(tokens, channelCounter, 40, pre, loc, cell)); - result.addAll(insert_bpms(tokens, channelCounter, 4, pre, "BSA", loc, cell, "small aperture BPM")); - result.addAll(insert_bpms(tokens, channelCounter, 4, pre, "BHS", loc, cell, "high stability BPM")); - result.addAll(insert_bpms(tokens, channelCounter, 4, pre, "BLA", loc, cell, "large aperture BPM")); + result.addAll(insertBpms(tokens, channelCounter, 4, pre, "BSA", loc, cell, "small aperture BPM")); + result.addAll(insertBpms(tokens, channelCounter, 4, pre, "BHS", loc, cell, "high stability BPM")); + result.addAll(insertBpms(tokens, channelCounter, 4, pre, "BLA", loc, cell, "large aperture BPM")); + return result; + } + + private void bulkInsertChannels(Collection result) throws IOException { long start = System.currentTimeMillis(); BulkRequest.Builder br = new BulkRequest.Builder(); for (Channel channel : result) { @@ -329,171 +373,143 @@ private boolean insertSRCell(String cell) throws Exception { BulkResponse srResult = client.bulk(br.build()); String execute = "|Execute: " + (System.currentTimeMillis() - start) + "|"; - logger.log(Level.INFO, "Insterted BO cell " + cell + " " + prepare + " " + execute); + logger.log(Level.INFO, "Inserted cell " + prepare + " " + execute); if (srResult.errors()) { - logger.log(Level.SEVERE, "Bulk had errors"); + logger.log(Level.SEVERE, "Bulk insert had errors"); for (BulkResponseItem item : srResult.items()) { if (item.error() != null) { logger.log(Level.SEVERE, () -> item.error().reason()); } } - } else { - return true; } - return false; } - private boolean insertBOCell(String cell) throws Exception { + private Collection insertBOCell(String cell) { String loc = "booster"; String pre = "BR:C"; // Tokens Map> tokens = new HashMap<>(); for (int i = 0; i < 6; i++) { - tokens.put(i, new ArrayList<>(tokens_500)); + tokens.put(i, new ArrayList<>(tokens500)); } AtomicInteger channelCounter = new AtomicInteger(0); Collection result = new ArrayList<>(500); - result.addAll(insert_big_magnets(tokens, channelCounter, 2, pre, "DP", loc, cell, "dipole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "QDP:D", loc, cell, "defocusing quadrupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "QDP:F", loc, cell, "focusing quadrupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 2, pre, "STP", loc, cell, "sextupole")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "HC", loc, cell, "horizontal corrector")); - result.addAll(insert_big_magnets(tokens, channelCounter, 4, pre, "VC", loc, cell, "vertical corrector")); + result.addAll(insertBigMagnets(tokens, channelCounter, 2, pre, "DP", loc, cell, "dipole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "QDP:D", loc, cell, "defocusing quadrupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "QDP:F", loc, cell, "focusing quadrupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 2, pre, "STP", loc, cell, "sextupole")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "HC", loc, cell, "horizontal corrector")); + result.addAll(insertBigMagnets(tokens, channelCounter, 4, pre, "VC", loc, cell, "vertical corrector")); - result.addAll(insert_valves(tokens, channelCounter, 4, pre, "GV", loc, cell, "vacuum")); + result.addAll(insertValves(tokens, channelCounter, 4, pre, loc, cell)); - result.addAll(insert_gauges(tokens, channelCounter, 4, pre, "VGC", loc, cell, "vacuum")); - result.addAll(insert_gauges(tokens, channelCounter, 2, pre, "TCG", loc, cell, "vacuum")); + result.addAll(insertGauges(tokens, channelCounter, 4, pre, "VGC", loc, cell)); + result.addAll(insertGauges(tokens, channelCounter, 2, pre, "TCG", loc, cell)); - result.addAll(insert_pumps(tokens, channelCounter, 2, pre, "IPC", loc, cell, "vacuum")); - result.addAll(insert_pumps(tokens, channelCounter, 2, pre, "TMP", loc, cell, "vacuum")); + result.addAll(insertPumps(tokens, channelCounter, pre, "IPC", loc, cell)); + result.addAll(insertPumps(tokens, channelCounter, pre, "TMP", loc, cell)); - result.addAll(insert_temps(tokens, channelCounter, 10, pre, "TC", loc, cell, "temperature sensor")); + result.addAll(insertTemps(tokens, channelCounter, 10, pre, loc, cell)); - result.addAll(insert_bpms(tokens, channelCounter, 2, pre, "BLA", loc, cell, "beam position monitor")); - - long start = System.currentTimeMillis(); - BulkRequest.Builder br = new BulkRequest.Builder(); - for (Channel channel : result) { - br.operations(op -> op.index(IndexOperation.of(i -> i.index(esService.getES_CHANNEL_INDEX()).id(channel.getName()).document(channel)))); - } - String prepare = "|Prepare: " + (System.currentTimeMillis() - start) + "|"; - start = System.currentTimeMillis(); - br.refresh(Refresh.True); - - BulkResponse boosterResult = client.bulk(br.build()); - String execute = "|Execute: " + (System.currentTimeMillis() - start) + "|"; - logger.log(Level.INFO, "Insterted BO cell " + cell + " " + prepare + " " + execute); - if (boosterResult.errors()) { - logger.log(Level.SEVERE, "insertBOCell Bulk had errors"); - for (BulkResponseItem item : boosterResult.items()) { - if (item.error() != null) { - logger.log(Level.SEVERE, () -> item.error().reason()); - } - } - } else { - return true; - } - return false; + result.addAll(insertBpms(tokens, channelCounter, 2, pre, "BLA", loc, cell, "beam position monitor")); + return result; } - private Collection insert_big_magnets(Map> tokens, AtomicInteger channelInCell, int count, String prefix, - String dev, String loc, String cell, String element) { - List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-RB", loc, cell, element, "power supply", "current", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-SP", loc, cell, element, "power supply", "current", "setpoint")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-Sw", loc, cell, element, "power supply", "power", "switch")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Rst-Cmd", loc, cell, element, "power supply", "reset", "command")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-St", loc, cell, element, "power supply", "power", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Acc-St", loc, cell, element, "power supply", "access", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}OK-St", loc, cell, element, "power supply", "sum error", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}T-St", loc, cell, element, "power supply", "temperature", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}F-St", loc, cell, element, "power supply", "water flow", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Gnd-St", loc, cell, element, "power supply", "ground", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Ctl-St", loc, cell, element, "power supply", "control", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Val-St", loc, cell, element, "power supply", "value", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-RB", loc, cell, element, "magnet", "field", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-SP", loc, cell, element, "magnet", "field", "setpoint")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T:1-RB", loc, cell, element, "magnet", "temperature", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T:2-RB", loc, cell, element, "magnet", "temperature", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F-RB", loc, cell, element, "magnet", "water flow", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:in-St", loc, cell, element, "magnet", "water flow in", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:out-St", loc, cell, element, "magnet", "water flow out", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:dif-St", loc, cell, element, "magnet", "water flow diff", "status")); + private Collection insertBigMagnets(Map> tokens, AtomicInteger channelInCell, int count, String prefix, + String dev, String loc, String cell, String element) { + ArrayList channels = new ArrayList<>(); + insertPowerSupplyChannels(channels, tokens, channelInCell, count, prefix, dev, loc, cell, element); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}T-St", loc, cell, element, DEVICE_POWER_SUPPLY, UNIT_TEMP, SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}F-St", loc, cell, element, DEVICE_POWER_SUPPLY, "water flow", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Gnd-St", loc, cell, element, DEVICE_POWER_SUPPLY, "ground", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Ctl-St", loc, cell, element, DEVICE_POWER_SUPPLY, "control", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Val-St", loc, cell, element, DEVICE_POWER_SUPPLY, "value", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-RB", loc, cell, element, DEVICE_MAGNET, UNIT_FIELD, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-SP", loc, cell, element, DEVICE_MAGNET, UNIT_FIELD, SIGTYPE_SETPOINT)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T:1-RB", loc, cell, element, DEVICE_MAGNET, UNIT_TEMP, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T:2-RB", loc, cell, element, DEVICE_MAGNET, UNIT_TEMP, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F-RB", loc, cell, element, DEVICE_MAGNET, "water flow", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:in-St", loc, cell, element, DEVICE_MAGNET, "water flow in", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:out-St", loc, cell, element, DEVICE_MAGNET, "water flow out", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}F:dif-St", loc, cell, element, DEVICE_MAGNET, "water flow diff", SIGTYPE_STATUS)); return channels; } - private Collection insert_air_magnets(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell, String element) { - List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-RB", loc, cell, element, "power supply", "current", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-SP", loc, cell, element, "power supply", "current", "setpoint")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-Sw", loc, cell, element, "power supply", "power", "switch")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Rst-Cmd", loc, cell, element, "power supply", "reset", "command")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-St", loc, cell, element, "power supply", "power", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Acc-St", loc, cell, element, "power supply", "access", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}OK-St", loc, cell, element, "power supply", "sum error", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-RB", loc, cell, element, "magnet", "field", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-SP", loc, cell, element, "magnet", "field", "setpoint")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T-RB", loc, cell, element, "magnet", "temperature", "readback")); + private void insertPowerSupplyChannels(ArrayList channels, Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell, String element) { + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-RB", loc, cell, element, DEVICE_POWER_SUPPLY, UNIT_CURRENT, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}I-SP", loc, cell, element, DEVICE_POWER_SUPPLY, UNIT_CURRENT, SIGTYPE_SETPOINT)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-Sw", loc, cell, element, DEVICE_POWER_SUPPLY, UNIT_POWER, SIGTYPE_SWITCH)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Rst-Cmd", loc, cell, element, DEVICE_POWER_SUPPLY, "reset", "command")); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}On-St", loc, cell, element, DEVICE_POWER_SUPPLY, UNIT_POWER, SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}Acc-St", loc, cell, element, DEVICE_POWER_SUPPLY, "access", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PS:", "{" + dev + "}OK-St", loc, cell, element, DEVICE_POWER_SUPPLY, "sum error", SIGTYPE_STATUS)); + } + + private Collection insertAirMagnets(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell, String element) { + ArrayList channels = new ArrayList<>(); + insertPowerSupplyChannels(channels, tokens, channelInCell, count, prefix, dev, loc, cell, element); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-RB", loc, cell, element, DEVICE_MAGNET, UNIT_FIELD, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}Fld-SP", loc, cell, element, DEVICE_MAGNET, UNIT_FIELD, SIGTYPE_SETPOINT)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "MG:", "{" + dev + "}T-RB", loc, cell, element, DEVICE_MAGNET, UNIT_TEMP, SIGTYPE_READBACK)); return channels; } - private Collection insert_valves(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell, String element) { - List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}Opn-Sw", loc, cell, element, "valve", "position", "switch")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}Opn-St", loc, cell, element, "valve", "position", "status")); + private Collection insertValves(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String loc, String cell) { + ArrayList channels = new ArrayList<>(); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "VA:", "{" + "GV" + "}Opn-Sw", loc, cell, PopulateService.ELEMENT_VACUUM, "valve", "position", SIGTYPE_SWITCH)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "VA:", "{" + "GV" + "}Opn-St", loc, cell, PopulateService.ELEMENT_VACUUM, "valve", "position", SIGTYPE_STATUS)); return channels; } - private Collection insert_gauges(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell, String element) { + private Collection insertGauges(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, String loc, String cell) { List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}P-RB", loc, cell, element, "gauge", "pressure", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}OK-St", loc, cell, element, "gauge", "error", "status")); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}P-RB", loc, cell, PopulateService.ELEMENT_VACUUM, "gauge", "pressure", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}OK-St", loc, cell, PopulateService.ELEMENT_VACUUM, "gauge", "error", SIGTYPE_STATUS)); return channels; } - private Collection insert_pumps(Map> tokens, AtomicInteger channelInCell, int count, String prefix, - String dev, String loc, String cell, String element) { + private Collection insertPumps(Map> tokens, AtomicInteger channelInCell, String prefix, + String dev, String loc, String cell) { List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}I-RB", loc, cell, element, "pump", "current", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}P-RB", loc, cell, element, "pump", "pressure", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}On-Sw", loc, cell, element, "pump", "power", "switch")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}OK-St", loc, cell, element, "pump", "error", "status")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "VA:", "{" + dev + "}On-St", loc, cell, element, "pump", "power", "status")); + channels.addAll(insertBunch(tokens, channelInCell, 2, prefix, "VA:", "{" + dev + "}I-RB", loc, cell, PopulateService.ELEMENT_VACUUM, "pump", UNIT_CURRENT, SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, 2, prefix, "VA:", "{" + dev + "}P-RB", loc, cell, PopulateService.ELEMENT_VACUUM, "pump", "pressure", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, 2, prefix, "VA:", "{" + dev + "}On-Sw", loc, cell, PopulateService.ELEMENT_VACUUM, "pump", UNIT_POWER, SIGTYPE_SWITCH)); + channels.addAll(insertBunch(tokens, channelInCell, 2, prefix, "VA:", "{" + dev + "}OK-St", loc, cell, PopulateService.ELEMENT_VACUUM, "pump", "error", SIGTYPE_STATUS)); + channels.addAll(insertBunch(tokens, channelInCell, 2, prefix, "VA:", "{" + dev + "}On-St", loc, cell, PopulateService.ELEMENT_VACUUM, "pump", UNIT_POWER, SIGTYPE_STATUS)); return channels; } - private Collection insert_temps(Map> tokens, AtomicInteger channelInCell, int count, String prefix, - String dev, String loc, String cell, String element) { + private Collection insertTemps(Map> tokens, AtomicInteger channelInCell, int count, String prefix, + String loc, String cell) { List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PU:T", "{" + dev + "}T:1-RB", loc, cell, element, "sensor", "temperature 1", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PU:T", "{" + dev + "}T:2-RB", loc, cell, element, "sensor", "temperature 2", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PU:T", "{" + dev + "}T:3-RB", loc, cell, element, "sensor", "temperature 3", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PU:T", "{" + dev + "}T:4-RB", loc, cell, element, "sensor", "temperature 4", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "PU:T", "{" + dev + "}On-St", loc, cell, element, "sensor", "power", "status")); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PU:T", "{" + "TC" + "}T:1-RB", loc, cell, ELEMENT_TEMPERATURE_SENSOR, DEVICE_SENSOR, "temperature 1", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PU:T", "{" + "TC" + "}T:2-RB", loc, cell, ELEMENT_TEMPERATURE_SENSOR, DEVICE_SENSOR, "temperature 2", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PU:T", "{" + "TC" + "}T:3-RB", loc, cell, ELEMENT_TEMPERATURE_SENSOR, DEVICE_SENSOR, "temperature 3", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PU:T", "{" + "TC" + "}T:4-RB", loc, cell, ELEMENT_TEMPERATURE_SENSOR, DEVICE_SENSOR, "temperature 4", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "PU:T", "{" + "TC" + "}On-St", loc, cell, ELEMENT_TEMPERATURE_SENSOR, DEVICE_SENSOR, UNIT_POWER, SIGTYPE_STATUS)); return channels; } - private Collection insert_bpms(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, - String loc, String cell, String element) { + private Collection insertBpms(Map> tokens, AtomicInteger channelInCell, int count, String prefix, String dev, + String loc, String cell, String element) { List channels = new ArrayList<>(); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Pos:X-RB", loc, cell, element, "bpm", "x position", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Pos:Y-RB", loc, cell, element, "bpm", "y position", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Sig:X-RB", loc, cell, element, "bpm", "x sigma", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Sig:Y-RB", loc, cell, element, "bpm", "y sigma", "readback")); - channels.addAll(insert_bunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}On-St", loc, cell, element, "bpm", "power", "status")); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Pos:X-RB", loc, cell, element, "bpm", "x position", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Pos:Y-RB", loc, cell, element, "bpm", "y position", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Sig:X-RB", loc, cell, element, "bpm", "x sigma", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}Sig:Y-RB", loc, cell, element, "bpm", "y sigma", SIGTYPE_READBACK)); + channels.addAll(insertBunch(tokens, channelInCell, count, prefix, "BI:", "{" + dev + "}On-St", loc, cell, element, "bpm", UNIT_POWER, SIGTYPE_STATUS)); return channels; } - private Collection insert_bunch(Map> tokens, AtomicInteger channelInCellCounter, int count, String prefix, - String midfix, String postfix, String location, String cell, String element, String device, String unit, - String sigtype) { + private Collection insertBunch(Map> tokens, AtomicInteger channelInCellCounter, int count, String prefix, + String midfix, String postfix, String location, String cell, String element, String device, String unit, + String sigtype) { int cw = count > 9 ? 2 : 1; List result = new ArrayList<>(count); for (int i = 1; i < count + 1; i++) { @@ -520,15 +536,15 @@ private Collection insert_bunch(Map> tokens, Ato channel.getProperties().add(new Property("z_pos_r", powner, pos_c)); if (postfix.endsWith("}T:1-RB")) { - channel.getProperties().add(new Property("mount", powner, "outside")); + channel.getProperties().add(new Property(PROPERTY_NAME_MOUNT, powner, "outside")); } else if (postfix.endsWith("}T:2-RB")) { - channel.getProperties().add(new Property("mount", powner, "inside")); + channel.getProperties().add(new Property(PROPERTY_NAME_MOUNT, powner, "inside")); } else if (postfix.endsWith("}T:3-RB")) { - channel.getProperties().add(new Property("mount", powner, "top")); + channel.getProperties().add(new Property(PROPERTY_NAME_MOUNT, powner, "top")); } else if (postfix.endsWith("}T:4-RB")) { - channel.getProperties().add(new Property("mount", powner, "bottom")); + channel.getProperties().add(new Property(PROPERTY_NAME_MOUNT, powner, "bottom")); } else { - channel.getProperties().add(new Property("mount", powner, "center")); + channel.getProperties().add(new Property(PROPERTY_NAME_MOUNT, powner, "center")); } for (Entry> entry : tokens.entrySet()) { @@ -572,7 +588,7 @@ private Collection insert_bunch(Map> tokens, Ato } if (channelInCell % 2 == 0) { - channel.getProperties().add(new Property("group7", powner, "500")); + channel.getProperties().add(new Property("group7", powner, String.valueOf(500))); channel.getTags().add(new Tag("group7_500", towner)); } else if (channelInCell <= 2 * 200) { channel.getProperties().add(new Property("group7", powner, "200")); @@ -604,7 +620,7 @@ private Collection insert_bunch(Map> tokens, Ato } if (channelInCell < 500) { - channel.getProperties().add(new Property("group8", powner, "500")); + channel.getProperties().add(new Property("group8", powner, String.valueOf(500))); channel.getTags().add(new Tag("group8_500", towner)); } else if (channelInCell < 500 + 200) { channel.getProperties().add(new Property("group8", powner, "200")); @@ -636,7 +652,7 @@ private Collection insert_bunch(Map> tokens, Ato } if (channelInCell >= 500) { - channel.getProperties().add(new Property("group9", powner, "500")); + channel.getProperties().add(new Property("group9", powner, String.valueOf(500))); channel.getTags().add(new Tag("group9_500", towner)); } else if (channelInCell >= 500 - 200) { channel.getProperties().add(new Property("group9", powner, "200")); @@ -667,11 +683,11 @@ private Collection insert_bunch(Map> tokens, Ato channel.getTags().add(new Tag("group9_0", towner)); } - for (int j = 20; j < max_prop; j++) { + for (int j = 20; j < maxProp; j++) { channel.getProperties().add(new Property("prop" + String.format("%02d", j), powner, channelInCell + "-" + String.format("%02d", j))); } - for (int k = 11; k < max_tag; k++) { + for (int k = 11; k < maxTag; k++) { channel.getTags().add(new Tag("tag" + String.format("%02d", k), towner)); } int cellCount = Integer.parseInt(cell); @@ -695,11 +711,11 @@ private Collection insert_bunch(Map> tokens, Ato channel.getTags().add(new Tag("tagone", towner)); } - channel_list.add(channel.getName()); - prop_list.addAll(channel.getProperties().stream().map(p -> + channelList.add(channel.getName()); + propertySet.addAll(channel.getProperties().stream().map(p -> new Property(p.getName(), p.getOwner()) ).collect(Collectors.toSet())); - tag_list.addAll(channel.getTags()); + tagSet.addAll(channel.getTags()); result.add(channel); } return result; diff --git a/src/test/java/org/phoebus/channelfinder/ChannelRepositorySearchIT.java b/src/test/java/org/phoebus/channelfinder/ChannelRepositorySearchIT.java index 853035a8..5a178381 100644 --- a/src/test/java/org/phoebus/channelfinder/ChannelRepositorySearchIT.java +++ b/src/test/java/org/phoebus/channelfinder/ChannelRepositorySearchIT.java @@ -23,8 +23,8 @@ import java.util.logging.Logger; import static java.lang.Math.min; -import static org.phoebus.channelfinder.example.PopulateService.val_bucket; -import static org.phoebus.channelfinder.example.PopulateService.val_bucket_size; +import static org.phoebus.channelfinder.example.PopulateService.valBucket; +import static org.phoebus.channelfinder.example.PopulateService.valBucketSize; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @WebMvcTest(ChannelRepository.class) @@ -34,7 +34,7 @@ class ChannelRepositorySearchIT { // Need at least 10 000 channels to test Elastic search beyond the 10 000 default result limit // So needs to be a minimum of 7 - private final int CELLS = 7; + private final int CELLS = 100; @Autowired ChannelRepository channelRepository; @Autowired @@ -88,12 +88,12 @@ void searchTest() { Assertions.assertEquals(result.getChannels().get(0).getName(), channelNames.get(0)); logger.log(Level.INFO, "Search for all channels via wildcards"); - searchName(2, "BR:C001-BI:2{BLA}Pos:?-RB"); + searchName(2,2, "BR:C001-BI:2{BLA}Pos:?-RB"); - searchName(4, "BR:C001-BI:?{BLA}Pos:*"); + searchName(4, 4, "BR:C001-BI:?{BLA}Pos:*"); logger.log(Level.INFO, "Search for all 1000 channels"); - searchName(1000 * CELLS, "SR*"); + searchName(min(1000 * CELLS, ELASTIC_LIMIT), 1000 * CELLS, "SR*"); logger.log(Level.INFO, "Search for all 1000 SR channels and all 500 booster channels"); long allCount = 1500 * CELLS; @@ -118,33 +118,32 @@ void searchTest() { assertSearchCount(allCount, (int) elasticDefaultCount, allCount, searchParameters); logger.log(Level.INFO, "Search for channels based on a tag"); - for (long id = 1; id < val_bucket.size(); id++) { + for (long id = 1; id < valBucket.size(); id++) { - for (int bucket_index = 0; bucket_index < val_bucket.size(); bucket_index++) { - checkGroup(val_bucket_size.get(bucket_index), "~tag", "group" + id + "_" + val_bucket.get(bucket_index)); - checkGroup(val_bucket_size.get(bucket_index), "group" + id, String.valueOf(val_bucket.get(bucket_index))); + for (int bucket_index = 0; bucket_index < valBucket.size(); bucket_index++) { + checkGroup(valBucketSize.get(bucket_index), "~tag", "group" + id + "_" + valBucket.get(bucket_index)); + checkGroup(valBucketSize.get(bucket_index), "group" + id, String.valueOf(valBucket.get(bucket_index))); } } } - private void searchName(int expected, String name) { + private void searchName(int expectedChannels, int expectedQueryCount, String name) { MultiValueMap searchParameters = new LinkedMultiValueMap<>(); searchParameters.add("~name", name); - assertSearchCount(expected, expected, expected, searchParameters); + assertSearchCount(expectedChannels, expectedChannels, expectedQueryCount, searchParameters); } private void assertSearchCount(long expectedResultCount, int expectedChannelsCount, long expectedQueryCount, MultiValueMap searchParameters) { - + logger.log(Level.INFO, "Search for " + searchParameters + " expected " + expectedResultCount + " results " + expectedChannelsCount + " channels " + expectedQueryCount + " queries"); // Act SearchResult result = channelRepository.search(searchParameters); - long countResult = channelRepository.count(searchParameters); // Assert Assertions.assertEquals(expectedResultCount, result.getCount()); Assertions.assertEquals(expectedChannelsCount, result.getChannels().size()); - Assertions.assertEquals(expectedQueryCount, countResult); + Assertions.assertEquals(expectedQueryCount, channelRepository.count(searchParameters)); } private void checkGroup(int bucket, String key, String value) { @@ -154,10 +153,10 @@ private void checkGroup(int bucket, String key, String value) { searchParameters.add(key, value); SearchResult result = channelRepository.search(searchParameters); - long countResult = channelRepository.count(searchParameters); Integer expectedCount = CELLS * bucket; - Assertions.assertEquals(expectedCount, Integer.valueOf(result.getChannels().size()), "Search: " + maptoString(searchParameters)); - Assertions.assertEquals(expectedCount, Integer.valueOf((int) countResult)); + logger.log(Level.INFO, "Search for " + maptoString(searchParameters) + " expected " + expectedCount + " results"); + Assertions.assertEquals(min(expectedCount, ELASTIC_LIMIT), Integer.valueOf(result.getChannels().size()), "Search: " + maptoString(searchParameters)); + Assertions.assertEquals(expectedCount, Integer.valueOf((int) channelRepository.count(searchParameters))); } private String maptoString(MultiValueMap searchParameters) {