From 7bac6446d5a0819fc158400faf9ba89e864e3bb6 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 2 Oct 2024 17:00:58 -0400 Subject: [PATCH] ENH: support plugin loading in conifg (#4974) --------- Signed-off-by: George Chen --- data-prepper-api/build.gradle | 2 +- .../annotations/UsesDataPrepperPlugin.java | 23 ++++++ .../plugin/ClasspathPluginProvider.java | 15 ++++ .../dataprepper/plugin/PluginProvider.java | 12 ++++ .../plugin/ClasspathPluginProviderTest.java | 54 ++++++++++++++ data-prepper-plugin-schema-cli/build.gradle | 1 + .../DataPrepperPluginSchemaExecute.java | 12 ++-- .../schemas/JsonSchemaConverter.java | 46 +++++++++++- .../PluginConfigsJsonSchemaConverter.java | 15 ++-- .../schemas/JsonSchemaConverterIT.java | 72 +++++++++++++++++++ .../schemas/JsonSchemaConverterTest.java | 17 +++-- .../PluginConfigsJsonSchemaConverterIT.java | 12 ++-- .../PluginConfigsJsonSchemaConverterTest.java | 10 +-- .../aggregate/AggregateProcessorConfig.java | 2 + .../AnomalyDetectorProcessorConfig.java | 2 + .../ObfuscationProcessorConfig.java | 3 + settings.gradle | 2 +- 17 files changed, 264 insertions(+), 36 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java create mode 100644 data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 045d331704..bf0f0aebd6 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -13,8 +13,8 @@ dependencies { implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' implementation libs.parquet.common - testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.commons.lang3 + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation project(':data-prepper-test-common') testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation libs.commons.io diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java new file mode 100644 index 0000000000..e94e2cca4c --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java @@ -0,0 +1,23 @@ +package org.opensearch.dataprepper.model.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotates a field that uses Data Prepper plugin config as its value. + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface UsesDataPrepperPlugin { + /** + * The class type for this plugin. + * + * @return The Java class + * @since 1.2 + */ + Class pluginType(); +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java index 764c83f4db..df29915cd1 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java @@ -18,6 +18,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.opensearch.dataprepper.model.annotations.DataPrepperPlugin.DEFAULT_ALTERNATE_NAME; import static org.opensearch.dataprepper.model.annotations.DataPrepperPlugin.DEFAULT_DEPRECATED_NAME; @@ -60,6 +61,20 @@ public Optional> findPluginClass(final Class pluginTyp return Optional.ofNullable((Class) supportedTypesMap.get(pluginType)); } + @Override + public Set> findPluginClasses(Class pluginType) { + if (nameToSupportedTypeToPluginType == null) { + nameToSupportedTypeToPluginType = scanForPlugins(); + } + + return nameToSupportedTypeToPluginType.values().stream() + .flatMap(supportedTypeToPluginType -> + supportedTypeToPluginType.entrySet().stream() + .filter(entry -> pluginType.equals(entry.getKey())) + .flatMap(entry -> Stream.of((Class) entry.getValue()))) + .collect(Collectors.toSet()); + } + private Map, Class>> scanForPlugins() { final Set> dataPrepperPluginClasses = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class); diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java index dd15176569..3aff8b0e3c 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugin; +import java.util.Collection; import java.util.Optional; /** @@ -27,4 +28,15 @@ public interface PluginProvider { * @since 1.2 */ Optional> findPluginClass(Class pluginType, String pluginName); + + /** + * Finds the Java classes for a specific pluginType. + * + * @param pluginType The type of plugin which is being supported. + * e.g. {@link org.opensearch.dataprepper.model.sink.Sink}. + * @param The type + * @return An {@link Collection} of Java classes for plugins + * @since 1.2 + */ + Collection> findPluginClasses(Class pluginType); } diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java index 6cda169636..56ec0f4167 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.sink.Sink; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.test.TestSink; @@ -18,13 +19,16 @@ import org.reflections.Reflections; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.BDDMockito.given; @@ -61,6 +65,22 @@ void findPlugin_should_scan_for_plugins() { .getTypesAnnotatedWith(DataPrepperPlugin.class); } + @Test + void findPlugins_should_scan_for_plugins() { + final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); + + then(reflections).shouldHaveNoInteractions(); + + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + objectUnderTest.findPluginClasses(Sink.class); + + then(reflections) + .should() + .getTypesAnnotatedWith(DataPrepperPlugin.class); + } + @Test void findPlugin_should_scan_for_plugins_only_once() { final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); @@ -76,6 +96,21 @@ void findPlugin_should_scan_for_plugins_only_once() { .getTypesAnnotatedWith(DataPrepperPlugin.class); } + @Test + void findPlugins_should_scan_for_plugins_only_once() { + final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); + + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + for (int i = 0; i < 10; i++) + objectUnderTest.findPluginClasses(Sink.class); + + then(reflections) + .should() + .getTypesAnnotatedWith(DataPrepperPlugin.class); + } + @Test void findPlugin_should_return_empty_if_no_plugins_found() { given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) @@ -130,6 +165,17 @@ void findPlugin_should_return_plugin_if_found_for_alternate_name_and_type_using_ assertThat(optionalPlugin.get(), equalTo(TestSource.class)); } + @Test + void findPlugins_should_return_empty_if_no_plugins_found() { + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + final Collection> foundPlugins = createObjectUnderTest().findPluginClasses( + PluginSetting.class); + assertThat(foundPlugins, notNullValue()); + assertThat(foundPlugins.isEmpty(), is(true)); + } + @Nested class WithPredefinedPlugins { @@ -161,5 +207,13 @@ void findPlugin_should_return_plugin_if_found_for_name_and_type_using_pluginType assertThat(optionalPlugin.isPresent(), equalTo(true)); assertThat(optionalPlugin.get(), equalTo(TestSink.class)); } + + @Test + void findPlugins_should_return_plugins_if_plugin_found_for_specified_type() { + final Set> foundPlugins = createObjectUnderTest().findPluginClasses(Source.class); + assertThat(foundPlugins, notNullValue()); + assertThat(foundPlugins.size(), equalTo(1)); + assertThat(foundPlugins.stream().iterator().next(), equalTo(TestSource.class)); + } } } \ No newline at end of file diff --git a/data-prepper-plugin-schema-cli/build.gradle b/data-prepper-plugin-schema-cli/build.gradle index fdcf46cf92..830a401ae5 100644 --- a/data-prepper-plugin-schema-cli/build.gradle +++ b/data-prepper-plugin-schema-cli/build.gradle @@ -9,6 +9,7 @@ application { dependencies { implementation project(':data-prepper-plugins') + implementation project(':data-prepper-plugin-framework') implementation project(':data-prepper-plugin-schema') implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'org.reflections:reflections:0.10.2' diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java index 75115eb7e6..fd817631a6 100644 --- a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java @@ -5,11 +5,9 @@ import com.github.victools.jsonschema.generator.SchemaVersion; import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -58,11 +56,9 @@ public void run() { new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); final PluginConfigsJsonSchemaConverter pluginConfigsJsonSchemaConverter = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules), siteUrl, siteBaseUrl); + pluginProvider, new JsonSchemaConverter(modules, pluginProvider), siteUrl, siteBaseUrl); final Class pluginType = pluginConfigsJsonSchemaConverter.pluginTypeNameToPluginType(pluginTypeName); final Map pluginNameToJsonSchemaMap = pluginConfigsJsonSchemaConverter.convertPluginConfigsIntoJsonSchemas( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, pluginType); diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 0985485952..7172bbbd02 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -10,16 +10,27 @@ import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart; +import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart; import com.github.victools.jsonschema.generator.SchemaVersion; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.opensearch.dataprepper.plugin.PluginProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; public class JsonSchemaConverter { + private static final Logger LOG = LoggerFactory.getLogger(JsonSchemaConverter.class); static final String DEPRECATED_SINCE_KEY = "deprecated"; private final List jsonSchemaGeneratorModules; + private final PluginProvider pluginProvider; - public JsonSchemaConverter(final List jsonSchemaGeneratorModules) { + public JsonSchemaConverter(final List jsonSchemaGeneratorModules, final PluginProvider pluginProvider) { this.jsonSchemaGeneratorModules = jsonSchemaGeneratorModules; + this.pluginProvider = pluginProvider; } public ObjectNode convertIntoJsonSchema( @@ -30,7 +41,9 @@ public ObjectNode convertIntoJsonSchema( loadJsonSchemaGeneratorModules(configBuilder); final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart = configBuilder.forFields(); overrideInstanceAttributeWithDeprecated(scopeSchemaGeneratorConfigPart); + overrideTargetTypeWithUsesDataPrepperPlugin(scopeSchemaGeneratorConfigPart); resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart); + overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset); final SchemaGeneratorConfig config = configBuilder.build(); final SchemaGenerator generator = new SchemaGenerator(config); @@ -52,6 +65,37 @@ private void overrideInstanceAttributeWithDeprecated( }); } + private void overrideTargetTypeWithUsesDataPrepperPlugin( + final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { + scopeSchemaGeneratorConfigPart.withTargetTypeOverridesResolver(field -> Optional + .ofNullable(field.getAnnotationConsideringFieldAndGetterIfSupported(UsesDataPrepperPlugin.class)) + .map(usesDataPrepperPlugin -> + pluginProvider.findPluginClasses(usesDataPrepperPlugin.pluginType()).stream()) + .map(stream -> stream.map(specificSubtype -> field.getContext().resolve(specificSubtype))) + .map(stream -> stream.collect(Collectors.toList())) + .orElse(null)); + } + + private void overrideDataPrepperPluginTypeAttribute( + final SchemaGeneratorGeneralConfigPart schemaGeneratorGeneralConfigPart, + final SchemaVersion schemaVersion, final OptionPreset optionPreset) { + schemaGeneratorGeneralConfigPart.withTypeAttributeOverride((node, scope, context) -> { + final DataPrepperPlugin dataPrepperPlugin = scope.getType().getErasedType() + .getAnnotation(DataPrepperPlugin.class); + if (dataPrepperPlugin != null) { + final ObjectNode propertiesNode = node.putObject("properties"); + try { + final ObjectNode schemaNode = this.convertIntoJsonSchema( + schemaVersion, optionPreset, dataPrepperPlugin.pluginConfigurationType()); + propertiesNode.set(dataPrepperPlugin.name(), schemaNode); + } catch (JsonProcessingException e) { + LOG.error("Encountered error retrieving JSON schema for {}", dataPrepperPlugin.name(), e); + throw new RuntimeException(e); + } + } + }); + } + private void resolveDefaultValueFromJsonProperty( final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { scopeSchemaGeneratorConfigPart.withDefaultResolver(field -> { diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java index b7f4c1a531..eb9c29dd63 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java @@ -1,6 +1,5 @@ package org.opensearch.dataprepper.schemas; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaVersion; @@ -10,7 +9,7 @@ import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.sink.Sink; import org.opensearch.dataprepper.model.source.Source; -import org.reflections.Reflections; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,15 +50,15 @@ public class PluginConfigsJsonSchemaConverter { private final String siteUrl; private final String siteBaseUrl; - private final Reflections reflections; + private final PluginProvider pluginProvider; private final JsonSchemaConverter jsonSchemaConverter; public PluginConfigsJsonSchemaConverter( - final Reflections reflections, + final PluginProvider pluginProvider, final JsonSchemaConverter jsonSchemaConverter, final String siteUrl, final String siteBaseUrl) { - this.reflections = reflections; + this.pluginProvider = pluginProvider; this.jsonSchemaConverter = jsonSchemaConverter; this.siteUrl = siteUrl == null ? SITE_URL_PLACEHOLDER : siteUrl; this.siteBaseUrl = siteBaseUrl == null ? SITE_BASE_URL_PLACEHOLDER : siteBaseUrl; @@ -90,8 +89,8 @@ public Map convertPluginConfigsIntoJsonSchemas( addPluginName(jsonSchemaNode, pluginName); addDocumentationLink(jsonSchemaNode, pluginName, pluginType); value = jsonSchemaNode.toPrettyString(); - } catch (JsonProcessingException e) { - LOG.error("Encountered error retrieving JSON schema for {}", pluginName); + } catch (final Exception e) { + LOG.error("Encountered error retrieving JSON schema for {}", pluginName, e); return Stream.empty(); } return Stream.of(Map.entry(entry.getKey(), value)); @@ -107,7 +106,7 @@ private Map> scanForPluginConfigs(final Class pluginType) { if (ConditionalRoute.class.equals(pluginType)) { return Map.of(CONDITIONAL_ROUTE_PROCESSOR_NAME, ConditionalRoute.class); } - return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + return pluginProvider.findPluginClasses(pluginType).stream() .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())) .collect(Collectors.toMap( diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java new file mode 100644 index 0000000000..7a3dca5991 --- /dev/null +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java @@ -0,0 +1,72 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.annotation.JsonClassDescription; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; +import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction; + +import java.util.List; + +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class JsonSchemaConverterIT { + static final String PROPERTIES_KEY = "properties"; + static final String ANY_OF_KEY = "anyOf"; + + private JsonSchemaConverter objectUnderTest; + + @BeforeEach + void setUp() { + final List modules = List.of( + new JacksonModule(RESPECT_JSONPROPERTY_REQUIRED), + new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, + JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) + ); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); + objectUnderTest = new JsonSchemaConverter(modules, pluginProvider); + } + + @Test + void testSubTypes() throws JsonProcessingException { + final ObjectNode jsonSchemaNode = objectUnderTest.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); + final JsonNode propertiesNode = jsonSchemaNode.at("/" + PROPERTIES_KEY); + assertThat(propertiesNode, instanceOf(ObjectNode.class)); + assertThat(propertiesNode.has("action"), is(true)); + final JsonNode actionNode = propertiesNode.at("/action"); + assertThat(actionNode.has(ANY_OF_KEY), is(true)); + final JsonNode anyOfNode = actionNode.at("/" + ANY_OF_KEY); + assertThat(anyOfNode, instanceOf(ArrayNode.class)); + anyOfNode.forEach(aggregateActionNode -> assertThat(aggregateActionNode.has(PROPERTIES_KEY), is(true))); + } + + @JsonClassDescription("test config") + static class TestConfig { + @JsonPropertyDescription("The aggregate action description") + @UsesDataPrepperPlugin(pluginType = AggregateAction.class) + private PluginModel action; + + public PluginModel getAction() { + return action; + } + } +} diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index 67cf0ac527..3100370685 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -10,6 +10,10 @@ import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaVersion; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; import java.util.Collections; @@ -20,15 +24,19 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +@ExtendWith(MockitoExtension.class) class JsonSchemaConverterTest { + @Mock + private PluginProvider pluginProvider; - public JsonSchemaConverter createObjectUnderTest(final List modules) { - return new JsonSchemaConverter(modules); + public JsonSchemaConverter createObjectUnderTest(final List modules, final PluginProvider pluginProvider) { + return new JsonSchemaConverter(modules, pluginProvider); } @Test void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingException { - final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList()); + final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( + Collections.emptyList(), pluginProvider); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); @@ -44,7 +52,8 @@ void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingExceptio @Test void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingException { final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( - Collections.singletonList(new CustomJacksonModule())); + Collections.singletonList(new CustomJacksonModule()), + pluginProvider); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java index 71e9bf5faa..d825a3472f 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java @@ -13,10 +13,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; import java.util.List; import java.util.Map; @@ -46,11 +44,9 @@ void setUp() { new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); objectUnderTest = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules), TEST_URL, TEST_BASE_URL); + pluginProvider, new JsonSchemaConverter(modules, pluginProvider), TEST_URL, TEST_BASE_URL); } @ParameterizedTest diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java index 3d1c1b585a..39c5629e0c 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java @@ -12,7 +12,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.reflections.Reflections; +import org.opensearch.dataprepper.plugin.PluginProvider; import java.util.Map; import java.util.Set; @@ -40,7 +40,7 @@ class PluginConfigsJsonSchemaConverterTest { private JsonSchemaConverter jsonSchemaConverter; @Mock - private Reflections reflections; + private PluginProvider pluginProvider; @InjectMocks private PluginConfigsJsonSchemaConverter objectUnderTest; @@ -67,7 +67,7 @@ void testPluginTypeNameToPluginTypeWithInValidInput() { @Test void testConvertPluginConfigsIntoJsonSchemasHappyPath() throws JsonProcessingException { - when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + when(pluginProvider.findPluginClasses(eq(TestPluginType.class))).thenReturn(Set.of(TestPlugin.class)); final ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); when(jsonSchemaConverter.convertIntoJsonSchema( any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenReturn(objectNode); @@ -84,7 +84,7 @@ void testConvertPluginConfigsIntoJsonSchemasHappyPath() throws JsonProcessingExc @Test void testConvertPluginConfigsIntoJsonSchemasWithError() throws JsonProcessingException { - when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + when(pluginProvider.findPluginClasses(eq(TestPluginType.class))).thenReturn(Set.of(TestPlugin.class)); final JsonProcessingException jsonProcessingException = mock(JsonProcessingException.class); when(jsonSchemaConverter.convertIntoJsonSchema( any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenThrow( @@ -96,7 +96,7 @@ void testConvertPluginConfigsIntoJsonSchemasWithError() throws JsonProcessingExc @DataPrepperPlugin( name = "test_plugin", pluginType = TestPluginType.class, pluginConfigurationType = TestPluginConfig.class) - static class TestPlugin { + static class TestPlugin extends TestPluginType { } diff --git a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java index d637739d48..1dd7a14479 100644 --- a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java +++ b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonProperty; @@ -37,6 +38,7 @@ public class AggregateProcessorConfig { @JsonPropertyDescription("The action to be performed on each group. One of the available aggregate actions must be provided.") @JsonProperty("action") @NotNull + @UsesDataPrepperPlugin(pluginType = AggregateAction.class) private PluginModel aggregateAction; @JsonPropertyDescription("When local_mode is set to true, the aggregation is performed locally on each Data Prepper node instead of forwarding events to a specific node based on the identification_keys using a hash function. Default is false.") diff --git a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java index 0eb59edc58..2122a0f318 100644 --- a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java +++ b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonProperty; @@ -23,6 +24,7 @@ public class AnomalyDetectorProcessorConfig { @JsonPropertyDescription("The ML algorithm (or model) used to detect anomalies. You must provide a mode. See random_cut_forest mode.") @JsonProperty("mode") @NotNull + @UsesDataPrepperPlugin(pluginType = AnomalyDetectorMode.class) private PluginModel detectorMode; @JsonPropertyDescription("A non-ordered List that is used as input to the ML algorithm to detect anomalies in the values of the keys in the list. At least one key is required.") diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java index 8a491a758b..f0cc686606 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java @@ -12,8 +12,10 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; +import org.opensearch.dataprepper.plugins.processor.obfuscation.action.ObfuscationAction; import java.util.List; @@ -40,6 +42,7 @@ public class ObfuscationProcessorConfig { @JsonProperty("action") @JsonPropertyDescription("The obfuscation action. Available actions include 'hash' and 'mask'.") + @UsesDataPrepperPlugin(pluginType = ObfuscationAction.class) private PluginModel action; @JsonProperty("obfuscate_when") diff --git a/settings.gradle b/settings.gradle index 4328fa9aac..ec5a86f255 100644 --- a/settings.gradle +++ b/settings.gradle @@ -54,7 +54,7 @@ dependencyResolutionManagement { library('bouncycastle-bcpkix', 'org.bouncycastle', 'bcpkix-jdk18on').versionRef('bouncycastle') version('guava', '32.1.2-jre') library('guava-core', 'com.google.guava', 'guava').versionRef('guava') - version('reflections', '0.9.12') + version('reflections', '0.10.2') library('reflections-core', 'org.reflections', 'reflections').versionRef('reflections') library('commons-lang3', 'org.apache.commons', 'commons-lang3').version('3.14.0') library('commons-io', 'commons-io', 'commons-io').version('2.15.1')