-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from trocco-io/feature/support-with_group_by
Add Support for AWS Cost Explorer's withGroupBy Feature
- Loading branch information
Showing
8 changed files
with
449 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
116 changes: 52 additions & 64 deletions
116
src/main/java/org/embulk/input/aws_cost_explorer/AwsCostExplorerInputPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
src/main/java/org/embulk/input/aws_cost_explorer/GroupsConfigValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package org.embulk.input.aws_cost_explorer; | ||
|
|
||
| import com.amazonaws.services.costexplorer.model.Dimension; | ||
| import com.amazonaws.services.costexplorer.model.GroupDefinitionType; | ||
| import org.embulk.config.ConfigException; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| public class GroupsConfigValidator | ||
| { | ||
| private GroupsConfigValidator() | ||
| { | ||
| } | ||
|
|
||
| /** | ||
| * Validates the groups configuration. | ||
| * | ||
| * @param groups the groups configuration | ||
| */ | ||
| public static void validate(List<Map<String, String>> groups) | ||
| { | ||
| if (groups.isEmpty()) { | ||
| return; | ||
| } | ||
|
|
||
| if (groups.size() > 2) { | ||
| throw new ConfigException("groups must have at most 2 elements."); | ||
| } | ||
|
|
||
| for (Map<String, String> group : groups) { | ||
| validateGroup(group); | ||
| } | ||
|
|
||
| validateDuplicateGroups(groups); | ||
| } | ||
|
|
||
| private static void validateGroup(Map<String, String> group) | ||
| { | ||
| final String groupType = group.get("type"); | ||
| final String groupKey = group.get("key"); | ||
|
|
||
| if (groupType == null || groupKey == null) { | ||
| throw new ConfigException("group must have a type and a key."); | ||
| } | ||
|
|
||
| if (GroupDefinitionType.DIMENSION.name().equals(groupType)) { | ||
| validateDimensionGroupKey(groupKey); | ||
| return; | ||
| } | ||
|
|
||
| for (GroupDefinitionType enumEntry : GroupDefinitionType.values()) { | ||
| if (enumEntry.toString().equals(groupType)) { | ||
| validateGroupKey(groupKey); | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| throw new ConfigException("group type must be one of the following: " | ||
| + Arrays.stream(GroupDefinitionType.values()).map(Enum::name).collect(Collectors.joining(", "))); | ||
| } | ||
|
|
||
| private static void validateDimensionGroupKey(String groupKey) | ||
| { | ||
| for (Dimension enumEntry : Dimension.values()) { | ||
| if (enumEntry.name().equals(groupKey)) { | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| throw new ConfigException("dimension group key must be one of the following: " | ||
| + Arrays.stream(Dimension.values()).map(Enum::name).collect(Collectors.joining(", "))); | ||
| } | ||
|
|
||
| private static void validateGroupKey(String groupKey) | ||
| { | ||
| if (!groupKey.isEmpty() && groupKey.length() <= 1024) { | ||
| return; | ||
| } | ||
|
|
||
| throw new ConfigException("group key must be non-empty and at most 1024 characters long."); | ||
| } | ||
|
|
||
| private static void validateDuplicateGroups(List<Map<String, String>> groups) | ||
| { | ||
| Set<String> typeAndKeyPairs = new HashSet<>(); | ||
|
|
||
| for (Map<String, String> group : groups) { | ||
| String type = group.get("type"); | ||
| String key = group.get("key"); | ||
| String pair = type + ":" + key; | ||
|
|
||
| if (!typeAndKeyPairs.add(pair)) { | ||
| throw new ConfigException("groups must not have duplicate type and key pairs."); | ||
| } | ||
| } | ||
| } | ||
| } |
73 changes: 73 additions & 0 deletions
73
src/main/java/org/embulk/input/aws_cost_explorer/client/AwsCostExplorerClient.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package org.embulk.input.aws_cost_explorer.client; | ||
|
|
||
| import com.amazonaws.services.costexplorer.AWSCostExplorer; | ||
| import com.amazonaws.services.costexplorer.model.GetCostAndUsageRequest; | ||
| import com.amazonaws.services.costexplorer.model.GetCostAndUsageResult; | ||
|
|
||
| import java.util.Spliterator; | ||
| import java.util.function.Consumer; | ||
| import java.util.stream.Stream; | ||
| import java.util.stream.StreamSupport; | ||
|
|
||
| public class AwsCostExplorerClient | ||
| { | ||
| private final AWSCostExplorer client; | ||
|
|
||
| public AwsCostExplorerClient(AWSCostExplorer client) | ||
| { | ||
| this.client = client; | ||
| } | ||
|
|
||
| /** | ||
| * Request all pages of the cost and usage data. | ||
| * | ||
| * @param requestParameters The request parameters for the cost and usage data. | ||
| * @return A stream of the cost and usage data. | ||
| */ | ||
| public Stream<AwsCostExplorerResponse> requestAll(GetCostAndUsageRequest requestParameters) | ||
| { | ||
| return StreamSupport.stream(makeResponseIterator(requestParameters), false); | ||
| } | ||
|
|
||
| private Spliterator<AwsCostExplorerResponse> makeResponseIterator(GetCostAndUsageRequest requestParameters) | ||
| { | ||
| return new Spliterator<AwsCostExplorerResponse>() | ||
| { | ||
| @Override | ||
| public boolean tryAdvance(Consumer<? super AwsCostExplorerResponse> action) | ||
| { | ||
| final AwsCostExplorerResponse response = request(requestParameters); | ||
| action.accept(response); | ||
|
|
||
| final String nextPageToken = response.getNextPageToken(); | ||
| requestParameters.setNextPageToken(nextPageToken); | ||
|
|
||
| return nextPageToken != null && !nextPageToken.isEmpty(); | ||
| } | ||
|
|
||
| private AwsCostExplorerResponse request(GetCostAndUsageRequest requestParameters) | ||
| { | ||
| final GetCostAndUsageResult result = client.getCostAndUsage(requestParameters); | ||
| return new AwsCostExplorerResponse(result); | ||
| } | ||
|
|
||
| @Override | ||
| public Spliterator<AwsCostExplorerResponse> trySplit() | ||
| { | ||
| return null; // Parallel processing is not supported. | ||
| } | ||
|
|
||
| @Override | ||
| public long estimateSize() | ||
| { | ||
| return Long.MAX_VALUE; // Size is unknown in advance. | ||
| } | ||
|
|
||
| @Override | ||
| public int characteristics() | ||
| { | ||
| return Spliterator.NONNULL | Spliterator.IMMUTABLE; | ||
| } | ||
| }; | ||
| } | ||
| } |
Oops, something went wrong.