Extract the data-prepper-plugin-framework from data-prepper-core (#4260)

Signed-off-by: Taylor Gray <tylgry@amazon.com>

build-resources.gradle

@@ -13,5 +13,6 @@ ext.coreProjects = [
         project(':data-prepper-pipeline-parser'),
         project(':data-prepper-plugins'),
         project(':data-prepper-test-common'),
-        project(':data-prepper-test-event')
+        project(':data-prepper-test-event'),
+        project(':data-prepper-plugin-framework')
 ]

data-prepper-core/build.gradle

@@ -19,6 +19,8 @@ dependencies {
     implementation project(':data-prepper-plugins:common')
     implementation project(':data-prepper-logstash-configuration')
     implementation project(':data-prepper-pipeline-parser')
+    implementation project(':data-prepper-plugin-framework')
+    testImplementation project(':data-prepper-plugin-framework').sourceSets.test.output
     testImplementation project(':data-prepper-plugins:common').sourceSets.test.output
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
@@ -28,7 +30,6 @@ dependencies {
     implementation 'io.micrometer:micrometer-registry-cloudwatch2'
     implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1'
     implementation 'software.amazon.awssdk:cloudwatch'
-    implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
     implementation platform('org.apache.logging.log4j:log4j-bom:2.22.1')
     implementation 'org.apache.logging.log4j:log4j-core'
     implementation 'org.apache.logging.log4j:log4j-slf4j2-impl'

data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java → data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java

@@ -15,9 +15,8 @@
 import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel;
 import org.opensearch.dataprepper.model.configuration.PluginSetting;
 import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
-import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;
 import org.opensearch.dataprepper.plugins.TestObjectPlugin;
-import org.opensearch.dataprepper.plugins.TestPlugin;
+import org.opensearch.dataprepper.plugins.test.TestPlugin;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 
 import java.util.HashMap;
@@ -39,7 +38,7 @@ class DefaultPluginFactoryIT {
     @Mock
     private PipelinesDataFlowModel pipelinesDataFlowModel;
     @Mock
-    private DataPrepperConfiguration dataPrepperConfiguration;
+    private ExtensionsConfiguration extensionsConfiguration;
     private String pluginName;
     private String objectPluginName;
     private String pipelineName;
@@ -62,7 +61,7 @@ private DefaultPluginFactory createObjectUnderTest() {
         coreContext.scan(DefaultAcknowledgementSetManager.class.getPackage().getName());
         coreContext.scan(DefaultPluginFactory.class.getPackage().getName());
         coreContext.register(PluginBeanFactoryProvider.class);
-        coreContext.registerBean(DataPrepperConfiguration.class, () -> dataPrepperConfiguration);
+        coreContext.registerBean(ExtensionsConfiguration.class, () -> extensionsConfiguration);
         coreContext.registerBean(PipelinesDataFlowModel.class, () -> pipelinesDataFlowModel);
         coreContext.refresh();
 

data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionsIT.java → data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/ExtensionsIT.java

@@ -3,10 +3,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.dataprepper.plugin;
+package org.opensearch.dataprepper.plugins;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.hamcrest.MatcherAssert;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -25,8 +26,9 @@
 import org.opensearch.dataprepper.parser.config.PipelineParserConfiguration;
 import org.opensearch.dataprepper.peerforwarder.PeerForwarderProvider;
 import org.opensearch.dataprepper.pipeline.router.RouterFactory;
-import org.opensearch.dataprepper.plugins.TestPluginUsingExtension;
-import org.opensearch.dataprepper.plugins.TestPluginUsingExtensionWithConfig;
+import org.opensearch.dataprepper.plugin.DefaultPluginFactory;
+import org.opensearch.dataprepper.plugin.ObjectMapperConfiguration;
+import org.opensearch.dataprepper.plugin.TestPluggableInterface;
 import org.opensearch.dataprepper.plugins.test.TestExtension;
 import org.opensearch.dataprepper.sourcecoordination.SourceCoordinatorFactory;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
@@ -114,7 +116,7 @@ void tearDown() {
 
     @Test
     void applyExtensions_creates_a_single_instance_of_the_extension() {
-        assertThat(TestExtension.getConstructedInstances(), equalTo(1));
+        MatcherAssert.assertThat(TestExtension.getConstructedInstances(), equalTo(1));
     }
 
     @Test

data-prepper-core/src/main/java/org/opensearch/dataprepper/parser/config/DataPrepperAppConfiguration.java

@@ -9,8 +9,8 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.opensearch.dataprepper.model.configuration.PluginModel;
 import org.opensearch.dataprepper.model.types.ByteCount;
-import org.opensearch.dataprepper.parser.ByteCountDeserializer;
-import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
 import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

data-prepper-core/src/main/java/org/opensearch/dataprepper/parser/model/DataPrepperConfiguration.java

@@ -16,6 +16,7 @@
 import org.opensearch.dataprepper.parser.config.MetricTagFilter;
 import org.opensearch.dataprepper.peerforwarder.PeerForwarderConfiguration;
 import org.opensearch.dataprepper.pipeline.PipelineShutdownOption;
+import org.opensearch.dataprepper.plugin.ExtensionsConfiguration;
 
 import java.time.Duration;
 import java.util.Collections;
@@ -28,7 +29,7 @@
 /**
  * Class to hold configuration for DataPrepper, including server port and Log4j settings
  */
-public class DataPrepperConfiguration {
+public class DataPrepperConfiguration implements ExtensionsConfiguration {
     static final Duration DEFAULT_SHUTDOWN_DURATION = Duration.ofSeconds(30L);
 
     private static final String DEFAULT_SOURCE_COORDINATION_STORE = "in_memory";
@@ -225,6 +226,7 @@ public PipelineShutdownOption getPipelineShutdown() {
         return pipelineShutdown;
     }
 
+    @Override
     public PipelineExtensions getPipelineExtensions() {
         return pipelineExtensions;
     }

data-prepper-core/src/test/java/org/opensearch/dataprepper/TestDataProvider.java

@@ -65,10 +65,6 @@ public class TestDataProvider {
     public static final String VALID_PEER_FORWARDER_WITH_ACM_SSL_CONFIG_FILE = "src/test/resources/valid_peer_forwarder_config_with_acm_ssl.yml";
     public static final String VALID_DATA_PREPPER_CONFIG_WITH_METRIC_FILTER = "src/test/resources/valid_data_prepper_config_with_metric_filter.yml";
     public static final String INVALID_DATA_PREPPER_CONFIG_WITH_METRIC_FILTER = "src/test/resources/invalid_data_prepper_config_with_metric_filter.yml";
-
-
-
-
     public static Set<String> VALID_MULTIPLE_PIPELINE_NAMES = new HashSet<>(Arrays.asList("test-pipeline-1",
             "test-pipeline-2", "test-pipeline-3"));
 }

data-prepper-core/src/test/java/org/opensearch/dataprepper/parser/model/DataPrepperConfigurationTests.java

@@ -7,9 +7,9 @@
 
 import org.opensearch.dataprepper.TestDataProvider;
 import org.opensearch.dataprepper.model.types.ByteCount;
-import org.opensearch.dataprepper.parser.ByteCountDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
 import org.opensearch.dataprepper.peerforwarder.PeerForwarderConfiguration;
-import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;

data-prepper-core/src/test/java/org/opensearch/dataprepper/parser/model/HeapCircuitBreakerConfigTest.java

@@ -11,8 +11,8 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.opensearch.dataprepper.model.types.ByteCount;
-import org.opensearch.dataprepper.parser.ByteCountDeserializer;
-import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
 
 import java.io.IOException;
 import java.io.InputStream;

data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/PeerForwarderConfigurationTest.java

@@ -9,7 +9,7 @@
 
 import org.opensearch.dataprepper.TestDataProvider;
 import org.opensearch.dataprepper.peerforwarder.discovery.DiscoveryMode;
-import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.exc.ValueInstantiationException;

data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/PipelineTests.java

@@ -30,8 +30,8 @@
 import org.opensearch.dataprepper.pipeline.router.Router;
 import org.opensearch.dataprepper.pipeline.router.RouterCopyRecordStrategy;
 import org.opensearch.dataprepper.pipeline.router.RouterGetRecordStrategy;
-import org.opensearch.dataprepper.plugins.TestSink;
-import org.opensearch.dataprepper.plugins.TestSource;
+import org.opensearch.dataprepper.plugins.test.TestSink;
+import org.opensearch.dataprepper.plugins.test.TestSource;
 import org.opensearch.dataprepper.plugins.TestSourceWithCoordination;
 import org.opensearch.dataprepper.plugins.TestSourceWithEnhancedCoordination;
 import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer;

data-prepper-pipeline-parser/build.gradle

@@ -1,3 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins:blocking-buffer')

data-prepper-core/src/main/java/org/opensearch/dataprepper/parser/ByteCountDeserializer.java → data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/ByteCountDeserializer.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.dataprepper.parser;
+package org.opensearch.dataprepper.pipeline.parser;
 
 import com.fasterxml.jackson.core.JacksonException;
 import com.fasterxml.jackson.core.JsonParser;

data-prepper-core/src/main/java/org/opensearch/dataprepper/parser/DataPrepperDurationDeserializer.java → data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/DataPrepperDurationDeserializer.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.dataprepper.parser;
+package org.opensearch.dataprepper.pipeline.parser;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.DeserializationContext;

data-prepper-core/src/test/java/org/opensearch/dataprepper/parser/ByteCountDeserializerTest.java → data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/ByteCountDeserializerTest.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.dataprepper.parser;
+package org.opensearch.dataprepper.pipeline.parser;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;

data-prepper-core/src/test/java/org/opensearch/dataprepper/parser/DataPrepperDurationDeserializerTest.java → data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/DataPrepperDurationDeserializerTest.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.dataprepper.parser;
+package org.opensearch.dataprepper.pipeline.parser;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;
@@ -14,8 +14,8 @@
 
 import java.time.Duration;
 
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class DataPrepperDurationDeserializerTest {

data-prepper-plugin-framework/build.gradle

@@ -0,0 +1,22 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+dependencies {
+    implementation project(':data-prepper-api')
+    implementation project(':data-prepper-pipeline-parser')
+    implementation libs.commons.lang3
+    implementation 'javax.inject:javax.inject:1'
+    implementation 'javax.annotation:javax.annotation-api:1.3.2'
+    implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
+    implementation(libs.spring.core) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+    implementation(libs.spring.context) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+    implementation libs.reflections.core
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+    testImplementation testLibs.mockito.inline
+}

data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginConfigObservable.java → data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginConfigObservable.java

@@ -6,8 +6,8 @@
 package org.opensearch.dataprepper.plugin;
 
 import org.opensearch.dataprepper.model.configuration.PluginSetting;
-import org.opensearch.dataprepper.model.plugin.PluginConfigObserver;
 import org.opensearch.dataprepper.model.plugin.PluginConfigObservable;
+import org.opensearch.dataprepper.model.plugin.PluginConfigObserver;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;

data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java → data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java

@@ -6,7 +6,6 @@
 package org.opensearch.dataprepper.plugin;
 
 import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel;
-import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -21,10 +20,10 @@ public class ExtensionPluginConfigurationResolver {
     private final Map<String, Object> dataPrepperConfigExtensionMap;
 
     @Inject
-    public ExtensionPluginConfigurationResolver(final DataPrepperConfiguration dataPrepperConfiguration,
+    public ExtensionPluginConfigurationResolver(final ExtensionsConfiguration extensionsConfiguration,
                                                 final PipelinesDataFlowModel pipelinesDataFlowModel) {
-        this.dataPrepperConfigExtensionMap = dataPrepperConfiguration.getPipelineExtensions() == null?
-                new HashMap<>() : new HashMap<>(dataPrepperConfiguration.getPipelineExtensions().getExtensionMap());
+        this.dataPrepperConfigExtensionMap = extensionsConfiguration.getPipelineExtensions() == null?
+                new HashMap<>() : new HashMap<>(extensionsConfiguration.getPipelineExtensions().getExtensionMap());
         combinedExtensionMap = new HashMap<>(dataPrepperConfigExtensionMap);
         if (pipelinesDataFlowModel.getPipelineExtensions() != null) {
             combinedExtensionMap.putAll(pipelinesDataFlowModel.getPipelineExtensions().getExtensionMap());

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExtensionsConfiguration.java

@@ -0,0 +1,7 @@
+package org.opensearch.dataprepper.plugin;
+
+import org.opensearch.dataprepper.model.configuration.PipelineExtensions;
+
+public interface ExtensionsConfiguration {
+    PipelineExtensions getPipelineExtensions();
+}

data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java → data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java

@@ -9,8 +9,8 @@
 import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.opensearch.dataprepper.model.types.ByteCount;
-import org.opensearch.dataprepper.parser.ByteCountDeserializer;
-import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
+import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
 import org.springframework.context.annotation.Bean;
 
 import javax.inject.Named;

data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java → data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java

@@ -6,12 +6,12 @@
 package org.opensearch.dataprepper.plugin;
 
 import com.fasterxml.jackson.core.type.TypeReference;
-import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
-import org.opensearch.dataprepper.model.configuration.PluginSetting;
-import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.validation.ConstraintViolation;
 import jakarta.validation.Validator;
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
+import org.opensearch.dataprepper.model.configuration.PluginSetting;
+import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
 import org.springframework.context.annotation.DependsOn;
 
 import javax.inject.Named;

data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java → data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java

@@ -5,14 +5,14 @@
 
 package org.opensearch.dataprepper.plugin;
 
-import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
-import org.opensearch.dataprepper.model.sink.Sink;
-import org.opensearch.dataprepper.model.source.Source;
-import org.opensearch.dataprepper.plugins.TestSink;
-import org.opensearch.dataprepper.plugins.TestSource;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
+import org.opensearch.dataprepper.model.sink.Sink;
+import org.opensearch.dataprepper.model.source.Source;
+import org.opensearch.dataprepper.plugins.test.TestSink;
+import org.opensearch.dataprepper.plugins.test.TestSource;
 import org.reflections.Reflections;
 
 import java.util.Arrays;

data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java → data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java

@@ -5,19 +5,20 @@
 
 package org.opensearch.dataprepper.plugin;
 
+import org.hamcrest.MatcherAssert;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.ArgumentCaptor;
 import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
 import org.opensearch.dataprepper.model.configuration.PipelineDescription;
 import org.opensearch.dataprepper.model.configuration.PluginSetting;
 import org.opensearch.dataprepper.model.plugin.NoPluginFoundException;
 import org.opensearch.dataprepper.model.plugin.PluginConfigObservable;
 import org.opensearch.dataprepper.model.sink.Sink;
-import org.opensearch.dataprepper.plugins.TestSink;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
+import org.opensearch.dataprepper.plugins.test.TestSink;
 import org.springframework.beans.factory.BeanFactory;
 
 import java.util.ArrayList;
@@ -396,7 +397,7 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found_with_corr
                     .willReturn(expectedInstance);
 
             assertThat(createObjectUnderTest().loadPlugin(baseClass, pluginSetting), equalTo(expectedInstance));
-            assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).deprecatedName(), equalTo(TEST_SINK_DEPRECATED_NAME));
+            MatcherAssert.assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).deprecatedName(), equalTo(TEST_SINK_DEPRECATED_NAME));
             verify(beanFactoryProvider).get();
         }
     }