-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'develop' into 'master'
Release 0.52.0-SNAPSHOT See merge request hercules/hercules!392
- Loading branch information
Showing
87 changed files
with
1,261 additions
and
225 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
...ink/src/main/java/ru/kontur/vostok/hercules/elastic/sink/index/MatchingIndexResolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| package ru.kontur.vostok.hercules.elastic.sink.index; | ||
|
|
||
| import ru.kontur.vostok.hercules.configuration.Sources; | ||
| import ru.kontur.vostok.hercules.elastic.sink.index.matching.ConfigParser; | ||
| import ru.kontur.vostok.hercules.elastic.sink.index.matching.IndexData; | ||
| import ru.kontur.vostok.hercules.protocol.Event; | ||
| import ru.kontur.vostok.hercules.util.parameter.Parameter; | ||
| import ru.kontur.vostok.hercules.util.properties.PropertiesUtil; | ||
|
|
||
| import java.util.Optional; | ||
| import java.util.Properties; | ||
|
|
||
| /** | ||
| * Resolves the index name by choice from file with predefined indices and corresponding tag values. | ||
| * <p> | ||
| * File format rules: | ||
| * <ul> | ||
| * <li> File content must be in JSON format. | ||
| * <li> The content consist of index data array. | ||
| * <li> Value of the field {@code index} is an index name. | ||
| * <li> Value of the field {@code tagMaps} is an array of tag maps. | ||
| * <li> Every tag map consist of set of tag-value pairs. | ||
| * <li> Tag value may be present as plain string or regular expressions. | ||
| * </ul> | ||
| * <p> | ||
| * File content sample: | ||
| * <pre>{@code | ||
| * [ | ||
| * { | ||
| * "index": "first-index", | ||
| * "tagMaps": [ | ||
| * { | ||
| * "some-tag": "some-value", | ||
| * "outer-tag/inner-tag": "inner-value" | ||
| * }, | ||
| * { | ||
| * "some-tag": "alternative-value" | ||
| * } | ||
| * ] | ||
| * }, | ||
| * { | ||
| * "index": "second-index", | ||
| * "tagMaps": [ | ||
| * { | ||
| * "some-tag": "for-second-index-value" | ||
| * }, | ||
| * { | ||
| * "special-tag": "special-value" | ||
| * } | ||
| * ] | ||
| * } | ||
| * ]}</pre> | ||
| * The event matches the index if the event contains all tags from at least one of index tag maps. <p> | ||
| * If the event matches one or more indices then the name of first of these indices is returned with {@link Optional} wrap. <p> | ||
| * If the event doesn't match all indices then the resolver returns empty {@link Optional}. <p> | ||
| * <br> | ||
| * Example for above file content sample: | ||
| * <li> for event with single tag {@code 'some-tag=for-second-index-value'} index {@code 'second-index'} will be resolved, | ||
| * <li> for event with single tag {@code 'special-tag=special-value'} index {@code 'second-index'} will be resolved, | ||
| * <li> for event with tags {@code 'some-tag=some-value'} and {@code 'outer-tag/inner-tag=inner-value'} | ||
| * without other tags, index {@code 'first-index'} will be resolved, | ||
| * <li> for event with tags {@code 'some-tag=alternative-value'} and {@code 'special-tag=special-value'} | ||
| * without other tags, index {@code 'first-index'} will be resolved, | ||
| * <li> for event with tags {@code 'some-tag=some-value'} and {@code 'special-tag=non-special-value'} | ||
| * without other tags, no index will be resolved. | ||
| * | ||
| * @author Petr Demenev | ||
| */ | ||
| public class MatchingIndexResolver extends IndexResolver { | ||
|
|
||
| private final IndexData[] indices; | ||
|
|
||
| public MatchingIndexResolver(Properties properties) { | ||
| super(properties); | ||
| String filePath = PropertiesUtil.get(Props.FILE_PATH, properties).get(); | ||
| indices = ConfigParser.parse(Sources.load(filePath)); | ||
| } | ||
|
|
||
| @Override | ||
| public Optional<String> resolve(Event event) { | ||
| for (IndexData index : indices) { | ||
| boolean matches = index.getTagMaps().stream().anyMatch(tagMap -> tagMap.matches(event)); | ||
| if (matches) { | ||
| return Optional.of(index.getIndex()); | ||
| } | ||
| } | ||
| return Optional.empty(); | ||
| } | ||
|
|
||
| private static class Props { | ||
| private static final Parameter<String> FILE_PATH = Parameter.stringParameter("file"). | ||
| withDefault("file://indices.json"). | ||
| build(); | ||
| } | ||
| } |
51 changes: 51 additions & 0 deletions
51
...ink/src/main/java/ru/kontur/vostok/hercules/elastic/sink/index/matching/ConfigParser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package ru.kontur.vostok.hercules.elastic.sink.index.matching; | ||
|
|
||
| import com.fasterxml.jackson.core.type.TypeReference; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.ObjectReader; | ||
| import ru.kontur.vostok.hercules.protocol.hpath.HPath; | ||
| import ru.kontur.vostok.hercules.util.Maps; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.util.Arrays; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.regex.Pattern; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * Configures MatchingIndexResolver. | ||
| * Parses raw index data array from JSON and transform its contents to usable object types | ||
| * | ||
| * @author Petr Demenev | ||
| */ | ||
| public class ConfigParser { | ||
|
|
||
| public static IndexData[] parse(InputStream in) { | ||
| ObjectMapper mapper = new ObjectMapper(); | ||
| ObjectReader reader = mapper.readerFor(new TypeReference<IndexDataModel[]>() {}); | ||
| IndexDataModel[] indices; | ||
| try { | ||
| indices = reader.readValue(in); | ||
| } catch (IOException ex) { | ||
| throw new IllegalArgumentException("Cannot parse index map", ex); | ||
| } | ||
| return Arrays.stream(indices). | ||
| map(i -> new IndexData(i.getIndex(), transform(i.getTagMaps()))). | ||
| toArray(IndexData[]::new); | ||
| } | ||
|
|
||
| private static List<TagMap> transform(List<Map<String, String>> list) { | ||
| return list.stream(). | ||
| map(sourceMap -> { | ||
| Map<HPath, Pattern> resultMap = new HashMap<>(Maps.effectiveHashMapCapacity(sourceMap.size())); | ||
| sourceMap.forEach((key, value) -> resultMap.put( | ||
| HPath.fromPath(key), | ||
| Pattern.compile(value == null ? "null" : value))); | ||
| return new TagMap(resultMap); | ||
| }). | ||
| collect(Collectors.toList()); | ||
| } | ||
| } |
Oops, something went wrong.