From 9452bddb29083b8475ace98ba9792740050cc8e2 Mon Sep 17 00:00:00 2001 From: shnrndk Date: Thu, 8 Jun 2023 11:08:54 +0530 Subject: [PATCH 01/47] Add SSL and Auth source support to Mongo data-sources. --- .../dataservices/common/DBConstants.java | 2 ++ .../core/description/config/MongoConfig.java | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/src/main/java/org/wso2/micro/integrator/dataservices/common/DBConstants.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/src/main/java/org/wso2/micro/integrator/dataservices/common/DBConstants.java index 0e3b0089a1..343ede65a1 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/src/main/java/org/wso2/micro/integrator/dataservices/common/DBConstants.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/src/main/java/org/wso2/micro/integrator/dataservices/common/DBConstants.java @@ -673,6 +673,8 @@ private MongoDB() { public static final String CONNECT_TIMEOUT = "mongoDB_connectTimeout"; public static final String MAX_WAIT_TIME = "mongoDB_maxWaitTime"; public static final String SOCKET_TIMEOUT = "mongoDB_socketTimeout"; + public static final String SSL_ENABLED = "mongoDB_ssl_enabled"; + public static final String AUTH_SOURCE = "mongoDB_auth_source"; public static final String CONNECTIONS_PER_HOST = "mongoDB_connectionsPerHost"; public static final String THREADS_ALLOWED_TO_BLOCK_CONN_MULTIPLIER = "mongoDB_threadsAllowedToBlockForConnectionMultiplier"; public static final String RESULT_COLUMN_NAME = "Document"; diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/description/config/MongoConfig.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/description/config/MongoConfig.java index a7974b35eb..1999e5b2a5 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/description/config/MongoConfig.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/description/config/MongoConfig.java @@ -143,6 +143,11 @@ private MongoClientOptions extractMongoOptions(Map properties) { builder.threadsAllowedToBlockForConnectionMultiplier( Integer.parseInt(threadsAllowedToBlockForConnectionMultiplier)); } + + String sslEnabled = (properties.get(DBConstants.MongoDB.SSL_ENABLED)); + if (Boolean.parseBoolean(sslEnabled)) { + builder.sslEnabled(true); + } return builder.build(); } @@ -169,17 +174,23 @@ private MongoCredential createCredential(Map properties) throws String authenticationType = properties.get(DBConstants.MongoDB.AUTHENTICATION_TYPE); String username = properties.get(DBConstants.MongoDB.USERNAME); String password = properties.get(DBConstants.MongoDB.PASSWORD); - String database = properties.get(DBConstants.MongoDB.DATABASE); + String authSource = properties.get(DBConstants.MongoDB.AUTH_SOURCE); + if (authSource == null || authSource.isEmpty()) { + // For MONGODB-CR, SCRAM-SHA-1, and SCRAM-SHA-256, PLAIN the default auth source is the database tyring to connect + // refer: https://docs.mongodb.com/ruby-driver/master/reference/authentication/ + // since database is mandatory, we will not address the case where DB is not defined. + authSource = properties.get(DBConstants.MongoDB.DATABASE); + } if (authenticationType != null) { switch (authenticationType) { case DBConstants.MongoDB.MongoAuthenticationTypes.PLAIN: - credential = MongoCredential.createPlainCredential(username, database, password.toCharArray()); + credential = MongoCredential.createPlainCredential(username, authSource, password.toCharArray()); break; case DBConstants.MongoDB.MongoAuthenticationTypes.SCRAM_SHA_1: - credential = MongoCredential.createScramSha1Credential(username, database, password.toCharArray()); + credential = MongoCredential.createScramSha1Credential(username, authSource, password.toCharArray()); break; case DBConstants.MongoDB.MongoAuthenticationTypes.MONGODB_CR: - credential = MongoCredential.createMongoCRCredential(username, database, password.toCharArray()); + credential = MongoCredential.createMongoCRCredential(username, authSource, password.toCharArray()); break; case DBConstants.MongoDB.MongoAuthenticationTypes.GSSAPI: credential = MongoCredential.createGSSAPICredential(username); From a2430df96617364382c52e0b93fecf001dac5138 Mon Sep 17 00:00:00 2001 From: chanikag Date: Mon, 26 Jun 2023 15:00:46 +0530 Subject: [PATCH 02/47] Support executing sequence at the MI server startup This is to introduce an automation mode to MI To start the MI in automation mode the capp name with a main sequence has to be provided Ex: sh micro-integrator.sh --car sampleCarCompositeExporter --- .../deployer/AppDeployerUtils.java | 1 + .../deployer/CarbonApplication.java | 9 ++++ .../config/ApplicationConfiguration.java | 9 ++++ .../application/deployer/config/Artifact.java | 11 ++++ .../MicroIntegratorSequenceController.java | 51 +++++++++++++++++++ .../initializer/ServiceBusConstants.java | 6 +++ .../initializer/ServiceBusInitializer.java | 26 ++++++++++ .../application/deployer/CappDeployer.java | 13 +++++ distribution/src/assembly/bin.xml | 8 +++ distribution/src/conf/sequence-observers.xml | 26 ++++++++++ distribution/src/scripts/micro-integrator.sh | 8 +++ 11 files changed, 168 insertions(+) create mode 100644 components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/MicroIntegratorSequenceController.java create mode 100644 distribution/src/conf/sequence-observers.xml diff --git a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/AppDeployerUtils.java b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/AppDeployerUtils.java index 4dc035483f..935e8b3c9c 100644 --- a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/AppDeployerUtils.java +++ b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/AppDeployerUtils.java @@ -262,6 +262,7 @@ public static Artifact populateArtifact(OMElement artifactEle) { // read top level attributes artifact.setName(readAttribute(artifactEle, Artifact.NAME)); artifact.setVersion(readAttribute(artifactEle, Artifact.VERSION)); + artifact.setMainSequence(readAttribute(artifactEle, Artifact.MAIN_SEQUENCE)); artifact.setType(readAttribute(artifactEle, Artifact.TYPE)); artifact.setServerRole(readAttribute(artifactEle, Artifact.SERVER_ROLE)); diff --git a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/CarbonApplication.java b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/CarbonApplication.java index c63d619059..652ac0bc70 100644 --- a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/CarbonApplication.java +++ b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/CarbonApplication.java @@ -31,6 +31,7 @@ public class CarbonApplication { private String appFilePath; private String appVersion; private boolean deploymentCompleted; + private String mainSequence; private ApplicationConfiguration appConfig; @@ -100,5 +101,13 @@ public boolean isDeploymentCompleted() { public void setDeploymentCompleted(boolean deploymentCompleted) { this.deploymentCompleted = deploymentCompleted; } + + public String getMainSequence() { + return mainSequence; + } + + public void setMainSequence(String mainSequence) { + this.mainSequence = mainSequence; + } } diff --git a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/ApplicationConfiguration.java b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/ApplicationConfiguration.java index 432fe878b9..a464852570 100644 --- a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/ApplicationConfiguration.java +++ b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/ApplicationConfiguration.java @@ -51,6 +51,7 @@ public class ApplicationConfiguration { private String appName; private String appVersion; + private String mainSequence; private org.wso2.micro.application.deployer.config.Artifact applicationArtifact; /** @@ -157,6 +158,7 @@ private void buildConfiguration(OMElement documentElement) throws CarbonExceptio } this.appName = appArtifact.getName(); this.appVersion = appArtifact.getVersion(); + this.setMainSequence(appArtifact.getMainSequence()); String[] serverRoles = AppDeployerUtils.readServerRoles(); List depsToRemove = new ArrayList(); @@ -194,4 +196,11 @@ private void handleException(String msg, Exception e) throws CarbonException { throw new CarbonException(msg, e); } + public String getMainSequence() { + return mainSequence; + } + + public void setMainSequence(String mainSequence) { + this.mainSequence = mainSequence; + } } diff --git a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/Artifact.java b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/Artifact.java index 2dcb69f707..4308dc2ae3 100644 --- a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/Artifact.java +++ b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/application/deployer/config/Artifact.java @@ -28,6 +28,8 @@ public class Artifact { public static final String NAME = "name"; public static final String TYPE = "type"; public static final String VERSION = "version"; + + public static final String MAIN_SEQUENCE = "mainSequence"; public static final String ARTIFACT_XML = "artifact.xml"; public static final String REG_INFO_XML = "registry-info.xml"; @@ -47,6 +49,7 @@ public class Artifact { private String extractedPath; private String runtimeObjectName; private String deploymentStatus; + private String mainSequence; private List dependencies; private List subArtifacts; @@ -155,6 +158,14 @@ public void setDeploymentStatus(String deploymentStatus) { this.deploymentStatus = deploymentStatus; } + public String getMainSequence() { + return mainSequence; + } + + public void setMainSequence(String mainSequence) { + this.mainSequence = mainSequence; + } + public static class Dependency { private String name; private String version; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/MicroIntegratorSequenceController.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/MicroIntegratorSequenceController.java new file mode 100644 index 0000000000..8f68bad973 --- /dev/null +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/MicroIntegratorSequenceController.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.micro.integrator.initializer; + +import org.apache.synapse.MessageContext; +import org.apache.synapse.SequenceFlowObserver; +import org.wso2.micro.core.util.CoreServerInitializerHolder; + +public class MicroIntegratorSequenceController implements SequenceFlowObserver { + + private String name; + + private String seqName; + + @Override + public void setName(String observerName) { + name = observerName; + } + + @Override + public void start(MessageContext messageContext, String observedSeq) { + if ("true".equals(messageContext.getProperty(ServiceBusConstants.AUTOMATION_MODE_INITIALIZED_PROPERTY))) { + seqName = observedSeq; + messageContext.setProperty(ServiceBusConstants.AUTOMATION_MODE_INITIALIZED_PROPERTY, "false"); + } + } + + @Override + public void complete(MessageContext messageContext, String observedSeq) { + if (observedSeq != null && observedSeq.equals(seqName)) { + CoreServerInitializerHolder coreServerInitializerHolder = CoreServerInitializerHolder.getInstance(); + coreServerInitializerHolder.shutdownGracefully(); + } + } + +} diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java index c64b9a9566..3dc9b0923e 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java @@ -127,5 +127,11 @@ public static final class RegistryStore { public static final String SYNAPSE_CONNECTOR_PACKAGE = "org.wso2.carbon.connector"; public static final String DISABLE_CONNECTOR_INIT_SYSTEM_PROPERTY = "esb.connector.startup.init.disable"; + // Constants used to start MI in automation mode + + public static final String AUTOMATION_MODE_CAR_NAME_SYSTEM_PROPERTY = "automation.mode.seq.car.name"; + + public static final String AUTOMATION_MODE_INITIALIZED_PROPERTY = "AUTOMATION_MODE_INITIALIZED"; + } diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java index 4d4f2374cd..0000b00d59 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.synapse.AbstractExtendedSynapseHandler; +import org.apache.synapse.MessageContext; import org.apache.synapse.ServerConfigurationInformation; import org.apache.synapse.ServerConfigurationInformationFactory; import org.apache.synapse.ServerContextInformation; @@ -35,6 +36,7 @@ import org.apache.synapse.core.SynapseEnvironment; import org.apache.synapse.debug.SynapseDebugInterface; import org.apache.synapse.debug.SynapseDebugManager; +import org.apache.synapse.mediators.base.SequenceMediator; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceRegistration; import org.osgi.service.cm.ConfigurationAdmin; @@ -47,11 +49,13 @@ import org.osgi.service.component.annotations.ReferencePolicy; import org.wso2.carbon.inbound.endpoint.EndpointListenerLoader; import org.wso2.carbon.securevault.SecretCallbackHandlerService; +import org.wso2.micro.application.deployer.CarbonApplication; import org.wso2.micro.core.Constants; import org.wso2.micro.core.ServerShutdownHandler; import org.wso2.micro.integrator.core.services.Axis2ConfigurationContextService; import org.wso2.micro.integrator.core.services.CarbonServerConfigurationService; import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; +import org.wso2.micro.integrator.initializer.deployment.application.deployer.CappDeployer; import org.wso2.micro.integrator.initializer.handler.ProxyLogHandler; import org.wso2.micro.integrator.initializer.handler.SynapseExternalPropertyConfigurator; import org.wso2.micro.integrator.initializer.handler.transaction.TransactionCountHandler; @@ -218,6 +222,20 @@ protected void activate(ComponentContext ctxt) { configurationManager);*/ // Start Inbound Endpoint Listeners EndpointListenerLoader.loadListeners(); + String injectCarName = System.getProperty(ServiceBusConstants.AUTOMATION_MODE_CAR_NAME_SYSTEM_PROPERTY); + if (injectCarName != null) { + String sequenceName = getMainSequenceName(injectCarName); + if (sequenceName == null) { + log.error("Invalid cApp name or main sequence name not found"); + } else { + MessageContext synCtx = synapseEnvironment.createMessageContext(); + SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration(). + getSequence(sequenceName); + synapseEnvironment.getSequenceObservers().add(new MicroIntegratorSequenceController()); + synCtx.setProperty(ServiceBusConstants.AUTOMATION_MODE_INITIALIZED_PROPERTY, "true"); + synCtx.getEnvironment().injectMessage(synCtx, seq); + } + } } catch (Exception e) { handleFatal("Couldn't initialize the ESB...", e); } catch (Throwable t) { @@ -228,6 +246,14 @@ protected void activate(ComponentContext ctxt) { } } + private String getMainSequenceName(String cappName) { + CarbonApplication capp = CappDeployer.getCarbonAppByName(cappName); + if (capp != null) { + return capp.getMainSequence(); + } + return null; + } + @Deactivate protected void deactivate(ComponentContext ctxt) { if (Objects.nonNull(transactionCountHandlerComponent)) { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java index e7fa635f20..de6a0d1114 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java @@ -279,6 +279,10 @@ private CarbonApplication buildCarbonApplication(String targetCAppPath, String c if (appVersion != null && !("").equals(appVersion)) { carbonApplication.setAppVersion(appVersion); } + String mainSeq = appConfig.getMainSequence(); + if (mainSeq != null && !("").equals(mainSeq)) { + carbonApplication.setMainSequence(mainSeq); + } return carbonApplication; } @@ -341,6 +345,15 @@ public static List getCarbonApps() { return Collections.unmodifiableList(cAppMap); } + public static CarbonApplication getCarbonAppByName(String cAppName) { + for (CarbonApplication capp : cAppMap) { + if (cAppName.equals(capp.getAppName())) { + return capp; + } + } + return null; + } + /** * Checks whether a given file is a jar or an aar file. * diff --git a/distribution/src/assembly/bin.xml b/distribution/src/assembly/bin.xml index 26e584566f..6920477e5a 100755 --- a/distribution/src/assembly/bin.xml +++ b/distribution/src/assembly/bin.xml @@ -509,6 +509,14 @@ 644 + + src/conf/sequence-observers.xml + wso2mi-${pom.version}/conf + sequence-observers.xml + true + 644 + + src/conf/security/cipher-text.properties wso2mi-${pom.version}/conf/security diff --git a/distribution/src/conf/sequence-observers.xml b/distribution/src/conf/sequence-observers.xml new file mode 100644 index 0000000000..e3302517dc --- /dev/null +++ b/distribution/src/conf/sequence-observers.xml @@ -0,0 +1,26 @@ + + + + + diff --git a/distribution/src/scripts/micro-integrator.sh b/distribution/src/scripts/micro-integrator.sh index 8457bd01c8..4446577018 100644 --- a/distribution/src/scripts/micro-integrator.sh +++ b/distribution/src/scripts/micro-integrator.sh @@ -136,11 +136,18 @@ do CMD="version" elif [ "$c" = "--restart" ] || [ "$c" = "-restart" ] || [ "$c" = "restart" ]; then CMD="restart" + elif [ "$c" = "--car" ] || [ "$c" = "-car" ] || [ "$c" = "car" ]; then + ARGUMENT="car" + OPTARG="$2" else args="$args $c" fi done +if [ "$ARGUMENT" = "car" ]; then + CAR_NAME="$OPTARG" +fi + if [ "$CMD" = "--debug" ]; then if [ "$PORT" = "" ]; then echo " Please specify the debug port after the --debug option" @@ -331,6 +338,7 @@ do -DenableLivenessProbe=true \ -DenableManagementApi=true \ -DskipStartupExtensions=false \ + -Dautomation.mode.seq.car.name="$CAR_NAME" \ -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \ -Dorg.ops4j.pax.logging.logReaderEnabled=false \ -Dorg.ops4j.pax.logging.eventAdminEnabled=false \ From 8b1000a6982f29aa6f79c01a4239a6e99ad1f672 Mon Sep 17 00:00:00 2001 From: chanikag Date: Wed, 19 Jul 2023 15:34:25 +0530 Subject: [PATCH 03/47] Change bat file --- .../integrator/initializer/ServiceBusInitializer.java | 2 +- distribution/src/scripts/micro-integrator.bat | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java index 0000b00d59..9b8bb3adc8 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java @@ -223,7 +223,7 @@ protected void activate(ComponentContext ctxt) { // Start Inbound Endpoint Listeners EndpointListenerLoader.loadListeners(); String injectCarName = System.getProperty(ServiceBusConstants.AUTOMATION_MODE_CAR_NAME_SYSTEM_PROPERTY); - if (injectCarName != null) { + if (injectCarName != null && !injectCarName.isEmpty()) { String sequenceName = getMainSequenceName(injectCarName); if (sequenceName == null) { log.error("Invalid cApp name or main sequence name not found"); diff --git a/distribution/src/scripts/micro-integrator.bat b/distribution/src/scripts/micro-integrator.bat index 57c2ac5290..7b048ea28a 100644 --- a/distribution/src/scripts/micro-integrator.bat +++ b/distribution/src/scripts/micro-integrator.bat @@ -101,6 +101,10 @@ if ""%1""==""stop"" goto stopServer if ""%1""==""-stop"" goto stopServer if ""%1""==""--stop"" goto stopServer +if ""%1""==""car"" goto setCar +if ""%1""==""-car"" goto setCar +if ""%1""==""--car"" goto setCar + shift goto setupArgs @@ -139,6 +143,9 @@ goto findJdk if "%OS%"=="Windows_NT" @setlocal if "%OS%"=="WINNT" @setlocal +:setCar +set CAR_NAME=%2 + rem ---------- Handle the SSL Issue with proper JDK version -------------------- rem find the version of the jdk :findJdk @@ -190,7 +197,7 @@ if "%profileSet%" == "false" ( set profile=-Dprofile=micro-integrator-default ) -set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof" -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% -Djava.endorsed.dirs=%JAVA_ENDORSED% -DandesConfig=broker.xml -Dcarbon.registry.root=/ -Dcarbon.home="%CARBON_HOME%" -Dlogfiles.home="%CARBON_HOME%\repository\logs" -Dwso2.server.standalone=true -Djava.command="%JAVA_HOME%\bin\java" -Djava.opts="%JAVA_OPTS%" -Djava.io.tmpdir="%CARBON_HOME%\tmp" -Dcatalina.base="%CARBON_HOME%\wso2\lib\tomcat" -Dwso2.carbon.xml="%CARBON_HOME%\conf\carbon.xml" -Dwso2.registry.xml="%CARBON_HOME%\conf\registry.xml" -Dwso2.user.mgt.xml="%CARBON_HOME%\conf\user-mgt.xml" -Dwso2.transports.xml="%CARBON_HOME%\conf\mgt-transports.xml" -Djava.util.logging.config.file="%CARBON_HOME%\conf\log4j.properties" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -DNonUserCoreMode=true -DNonRegistryMode=true -Dcarbon.logs.path="%CARBON_HOME%\repository\logs" -Dcomponents.repo="%CARBON_HOME%\wso2\components\plugins" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -Dcarbon.components.dir.path="%CARBON_HOME%\wso2\components" -Dcarbon.dropins.dir.path="%CARBON_HOME%\dropins" -Dcarbon.external.lib.dir.path="%CARBON_HOME%\lib" -Dcarbon.patches.dir.path="%CARBON_HOME%\patches" -Dcarbon.internal.lib.dir.path="%CARBON_HOME%\wso2\lib" -Dconf.location="%CARBON_HOME%\conf" -Dcom.atomikos.icatch.file="%CARBON_HOME%\wso2\lib\transactions.properties" -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler -Dcom.atomikos.icatch.hide_init_file_path="true" -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dcom.sun.jndi.ldap.connect.pool.authentication=simple -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 -Dorg.terracotta.quartz.skipUpdateCheck=true -Dcarbon.classpath=%CARBON_CLASSPATH% -Dfile.encoding=UTF8 -Dlogger.server.name="micro-integrator" -Dqpid.conf="\conf\advanced" -Dproperties.file.path=default -DavoidConfigHashRead=true -DenableReadinessProbe=true -DenableLivenessProbe=true -DenableManagementApi=true -DskipStartupExtensions=false -Dlog4j2.contextSelector="org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" -Dorg.ops4j.pax.logging.logReaderEnabled=false -Dorg.ops4j.pax.logging.eventAdminEnabled=false %JAVA_VER_BASED_OPTS% %profile% -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" +set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof" -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% -Djava.endorsed.dirs=%JAVA_ENDORSED% -DandesConfig=broker.xml -Dcarbon.registry.root=/ -Dcarbon.home="%CARBON_HOME%" -Dlogfiles.home="%CARBON_HOME%\repository\logs" -Dwso2.server.standalone=true -Djava.command="%JAVA_HOME%\bin\java" -Djava.opts="%JAVA_OPTS%" -Djava.io.tmpdir="%CARBON_HOME%\tmp" -Dcatalina.base="%CARBON_HOME%\wso2\lib\tomcat" -Dwso2.carbon.xml="%CARBON_HOME%\conf\carbon.xml" -Dwso2.registry.xml="%CARBON_HOME%\conf\registry.xml" -Dwso2.user.mgt.xml="%CARBON_HOME%\conf\user-mgt.xml" -Dwso2.transports.xml="%CARBON_HOME%\conf\mgt-transports.xml" -Djava.util.logging.config.file="%CARBON_HOME%\conf\log4j.properties" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -DNonUserCoreMode=true -DNonRegistryMode=true -Dcarbon.logs.path="%CARBON_HOME%\repository\logs" -Dcomponents.repo="%CARBON_HOME%\wso2\components\plugins" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -Dcarbon.components.dir.path="%CARBON_HOME%\wso2\components" -Dcarbon.dropins.dir.path="%CARBON_HOME%\dropins" -Dcarbon.external.lib.dir.path="%CARBON_HOME%\lib" -Dcarbon.patches.dir.path="%CARBON_HOME%\patches" -Dcarbon.internal.lib.dir.path="%CARBON_HOME%\wso2\lib" -Dconf.location="%CARBON_HOME%\conf" -Dcom.atomikos.icatch.file="%CARBON_HOME%\wso2\lib\transactions.properties" -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler -Dcom.atomikos.icatch.hide_init_file_path="true" -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dcom.sun.jndi.ldap.connect.pool.authentication=simple -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 -Dorg.terracotta.quartz.skipUpdateCheck=true -Dcarbon.classpath=%CARBON_CLASSPATH% -Dfile.encoding=UTF8 -Dlogger.server.name="micro-integrator" -Dqpid.conf="\conf\advanced" -Dproperties.file.path=default -DavoidConfigHashRead=true -DenableReadinessProbe=true -DenableLivenessProbe=true -DenableManagementApi=true -DskipStartupExtensions=false -Dautomation.mode.seq.car.name="%CAR_NAME%" -Dlog4j2.contextSelector="org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" -Dorg.ops4j.pax.logging.logReaderEnabled=false -Dorg.ops4j.pax.logging.eventAdminEnabled=false %JAVA_VER_BASED_OPTS% %profile% -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" :runJava rem echo JAVA_HOME environment variable is set to %JAVA_HOME% From 4a3f395345589381279e30c045a3e1ad3e3acad0 Mon Sep 17 00:00:00 2001 From: chanikag Date: Mon, 7 Aug 2023 09:15:02 +0530 Subject: [PATCH 04/47] Add Openmetric reporting for MI dataservices This is to fix https://github.com/wso2/api-manager/issues/2032 To run, On Windows: micro-integrator.bat -DenablePrometheusApi For Linux/MacOS/CentOS : ./micro-integrator.sh -DenablePrometheusApi --- .../core/DBInOutMessageReceiver.java | 6 ++ .../metric/handler/DSMetricHandler.java | 57 +++++++++++++ .../metric/handler/MetricHandler.java | 49 +---------- .../reporter/PrometheusReporter.java | 66 ++++++++++++++- .../observability/util/MetricConstants.java | 14 ++++ .../observability/util/MetricUtils.java | 81 +++++++++++++++++++ .../pom.xml | 4 + .../initializer/ServiceBusConstants.java | 5 ++ .../initializer/ServiceBusInitializer.java | 30 +++++++ pom.xml | 5 ++ 10 files changed, 269 insertions(+), 48 deletions(-) create mode 100644 components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/DSMetricHandler.java create mode 100644 components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricUtils.java diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DBInOutMessageReceiver.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DBInOutMessageReceiver.java index fe7b351a35..1e7520d8e3 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DBInOutMessageReceiver.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DBInOutMessageReceiver.java @@ -38,6 +38,8 @@ public class DBInOutMessageReceiver extends RawXMLINOutMessageReceiver { private static final Log log = LogFactory.getLog(DBInOutMessageReceiver.class); + + private static final String DATA_SERVICE_LATENCY_TIMER = "DATA_SERVICE_LATENCY_TIMER"; /** * Invokes the business logic invocation on the service implementation class @@ -94,6 +96,10 @@ public void invokeBusinessLogic(MessageContext msgContext, msgContext.setProperty(Constants.FAULT_NAME, DBConstants.DS_FAULT_NAME); throw DBUtils.createAxisFault(e); } finally { + if (msgContext.getProperty(DATA_SERVICE_LATENCY_TIMER) != null) { + newMsgContext.setProperty(DATA_SERVICE_LATENCY_TIMER, + msgContext.getProperty(DATA_SERVICE_LATENCY_TIMER)); + } if (log.isDebugEnabled()) { String response; if (msgContext.getProperty(Constants.FAULT_NAME) != null && diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/DSMetricHandler.java b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/DSMetricHandler.java new file mode 100644 index 0000000000..f249c89af5 --- /dev/null +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/DSMetricHandler.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.micro.integrator.observability.metric.handler; + +import org.apache.axis2.AxisFault; +import org.apache.axis2.context.MessageContext; +import org.apache.axis2.handlers.AbstractHandler; +import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; +import org.wso2.micro.integrator.observability.util.MetricConstants; +import org.wso2.micro.integrator.observability.util.MetricUtils; + +public class DSMetricHandler extends AbstractHandler { + + private MetricReporter metricReporter; + public DSMetricHandler() { + metricReporter = MetricUtils.getMetricReporter(); + } + + @Override + public InvocationResponse invoke(MessageContext messageContext) throws AxisFault { + if (MicroIntegratorBaseUtils.isDataService(messageContext)) { + String dataServiceName = messageContext.getAxisService().getName(); + if (messageContext.isProcessingFault()) { + metricReporter.incrementCount(MetricConstants.DATA_SERVICE_REQUEST_COUNT_ERROR_TOTAL, + new String[]{dataServiceName, MetricConstants.DATA_SERVICE}); + metricReporter.observeTime(messageContext.getProperty(MetricConstants.DATA_SERVICE_LATENCY_TIMER)); + } else if (MetricConstants.MESSAGE_DIRECTION_IN.equalsIgnoreCase( + messageContext.getAxisMessage().getDirection())) { + metricReporter.incrementCount(MetricConstants.DATA_SERVICE_REQUEST_COUNT_TOTAL, + new String[]{dataServiceName, MetricConstants.DATA_SERVICE}); + messageContext.setProperty(MetricConstants.DATA_SERVICE_LATENCY_TIMER, + metricReporter.getTimer(MetricConstants.DATA_SERVICE_LATENCY_SECONDS, + new String[]{dataServiceName, + MetricConstants.DATA_SERVICE})); + } else if (MetricConstants.MESSAGE_DIRECTION_OUT.equalsIgnoreCase( + messageContext.getAxisMessage().getDirection())) { + metricReporter.observeTime(messageContext.getProperty(MetricConstants.DATA_SERVICE_LATENCY_TIMER)); + } + } + return InvocationResponse.CONTINUE; + } +} diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/MetricHandler.java b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/MetricHandler.java index d162422ef8..7d175e4854 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/MetricHandler.java +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/MetricHandler.java @@ -29,13 +29,10 @@ import org.apache.synapse.rest.RESTConstants; import org.apache.synapse.rest.RESTUtils; import org.apache.synapse.transport.nhttp.NhttpConstants; -import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.integrator.core.internal.MicroIntegratorBaseConstants; import org.wso2.micro.integrator.core.services.CarbonServerConfigurationService; -import org.wso2.micro.integrator.observability.metric.handler.prometheus.reporter.PrometheusReporter; import org.wso2.micro.integrator.observability.util.MetricConstants; - -import java.util.Map; +import org.wso2.micro.integrator.observability.util.MetricUtils; /** * Class for extracting metric information by wrapping the implementation and @@ -44,7 +41,6 @@ public class MetricHandler extends AbstractExtendedSynapseHandler { private static Log log = LogFactory.getLog(MetricHandler.class); - private static final String METRIC_REPORTER = "metric_reporter"; private static final String DELIMITER = "/"; private static final String EMPTY = ""; @@ -61,7 +57,7 @@ public class MetricHandler extends AbstractExtendedSynapseHandler { @Override public boolean handleServerInit() { - metricReporterInstance = this.getMetricReporter(); + metricReporterInstance = MetricUtils.getMetricReporter(); CarbonServerConfigurationService serverConfig = CarbonServerConfigurationService.getInstance(); String miVersion = serverConfig.getServerVersion(); String updateLevel = System.getProperty(MetricConstants.UPDATE_LEVEL); @@ -71,47 +67,6 @@ public boolean handleServerInit() { return true; } - /** - * Load the MetricReporter class from the deployment.toml file if a user has defined a MetricReporter. - * Use default PrometheusReporter if the user hasn't defined a MetricReporter or an error occurs - * during custom MetricReporter class invocation. - */ - private MetricReporter getMetricReporter() { - Map configs = ConfigParser.getParsedConfigs(); - Object metricReporterClass = configs.get(MetricConstants.METRIC_HANDLER + "." + METRIC_REPORTER); - Class loadedMetricClass; - MetricReporter reporterInstance; - - if (metricReporterClass != null) { - try { - loadedMetricClass = Class.forName(metricReporterClass.toString()); - reporterInstance = (MetricReporter) loadedMetricClass.newInstance(); - if (log.isDebugEnabled()) { - log.debug("The class " + metricReporterClass + " loaded successfully"); - } - } catch (IllegalAccessException | ClassNotFoundException | InstantiationException e) { - log.error("Error in loading the class " + metricReporterClass.toString() + - " .Hence loading the default PrometheusReporter class ", e); - reporterInstance = loadDefaultPrometheusReporter(); - } - } else { - reporterInstance = loadDefaultPrometheusReporter(); - } - return reporterInstance; - } - - /** - * Load the PrometheusReporter class by default. - */ - private MetricReporter loadDefaultPrometheusReporter() { - MetricReporter reporterInstance = new PrometheusReporter(); - if (log.isDebugEnabled()) { - log.debug("The class org.wso2.micro.integrator.obsrvability.handler.metrics.publisher.prometheus." + - "reporter.PrometheusReporter was loaded successfully"); - } - return reporterInstance; - } - @Override public boolean handleRequestInFlow(MessageContext synCtx) { synCtx.setProperty(RESTConstants.IS_PROMETHEUS_ENGAGED, null); diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/prometheus/reporter/PrometheusReporter.java b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/prometheus/reporter/PrometheusReporter.java index bc7c3f719b..c5844c8955 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/prometheus/reporter/PrometheusReporter.java +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/metric/handler/prometheus/reporter/PrometheusReporter.java @@ -41,13 +41,16 @@ public class PrometheusReporter implements MetricReporter { private Counter TOTAL_REQUESTS_RECEIVED_PROXY_SERVICE; private Counter TOTAL_REQUESTS_RECEIVED_API; private Counter TOTAL_REQUESTS_RECEIVED_INBOUND_ENDPOINT; + private Counter TOTAL_REQUESTS_RECEIVED_DATA_SERVICE; private Counter ERROR_REQUESTS_RECEIVED_PROXY_SERVICE; private Counter ERROR_REQUESTS_RECEIVED_API; private Counter ERROR_REQUESTS_RECEIVED_INBOUND_ENDPOINT; + private Counter ERROR_REQUESTS_RECEIVED_DATA_SERVICE; private Histogram PROXY_LATENCY_HISTOGRAM; private Histogram API_LATENCY_HISTOGRAM; private Histogram INBOUND_ENDPOINT_LATENCY_HISTOGRAM; + private Histogram DATA_SERVICE_LATENCY_HISTOGRAM; private Gauge SERVER_UP; private Gauge SERVICE_UP; @@ -58,6 +61,7 @@ public class PrometheusReporter implements MetricReporter { private double[] proxyLatencyBuckets; private double[] apiLatencyBuckets; private double[] inboundEndpointLatencyBuckets; + private double[] dataServiceLatencyBuckets; private Map metricMap = new HashMap(); @@ -70,6 +74,7 @@ public void initMetrics() { this.initializeProxyMetrics(); this.initializeApiMetrics(); this.initializeInboundEndpointMetrics(); + this.initializeDataServiceMetrics(); } @Override @@ -79,6 +84,7 @@ public void createMetrics(String serviceType, String type, String metricName, St proxyLatencyBuckets = new double[]{0.19, 0.20, 0.25, 0.30, 0.35, 0.40, 0.50, 0.60, 1, 5}; apiLatencyBuckets = new double[]{0.19, 0.20, 0.25, 0.30, 0.35, 0.40, 0.50, 0.60, 1, 5}; inboundEndpointLatencyBuckets = new double[]{0.19, 0.20, 0.25, 0.30, 0.35, 0.40, 0.50, 0.60, 1, 5}; + dataServiceLatencyBuckets = new double[]{0.19, 0.20, 0.25, 0.30, 0.35, 0.40, 0.50, 0.60, 1, 5}; Map configs = ConfigParser.getParsedConfigs(); createBuckets(configs); @@ -132,6 +138,21 @@ public void createMetrics(String serviceType, String type, String metricName, St .register(); metricMap.put(metricName, INBOUND_ENDPOINT_LATENCY_HISTOGRAM); } + } else if (serviceType.equalsIgnoreCase(SERVICE.DATA_SERVICE.name())) { + if (type.equals(MetricConstants.COUNTER)) { + TOTAL_REQUESTS_RECEIVED_DATA_SERVICE = Counter.build + (MetricConstants.DATA_SERVICE_REQUEST_COUNT_TOTAL, metricHelp). + labelNames(labels).register(); + metricMap.put(metricName, TOTAL_REQUESTS_RECEIVED_DATA_SERVICE); + } else if (type.equals(MetricConstants.HISTOGRAM)) { + DATA_SERVICE_LATENCY_HISTOGRAM = Histogram.build() + .name(MetricConstants.DATA_SERVICE_LATENCY_SECONDS) + .help(metricHelp) + .labelNames(labels) + .buckets(dataServiceLatencyBuckets) + .register(); + metricMap.put(metricName, DATA_SERVICE_LATENCY_HISTOGRAM); + } } else if (serviceType.equals(MetricConstants.SERVER)) { SERVER_UP = Gauge.build(MetricConstants.SERVER_UP, "Server status"). labelNames(labels).register(); @@ -168,6 +189,12 @@ public void initErrorMetrics(String serviceType, String type, String metricName, register(); metricMap.put(metricName, ERROR_REQUESTS_RECEIVED_INBOUND_ENDPOINT); + } else if (serviceType.equals(SERVICE.DATA_SERVICE.name())) { + ERROR_REQUESTS_RECEIVED_DATA_SERVICE = Counter. + build(MetricConstants.DATA_SERVICE_REQUEST_COUNT_ERROR_TOTAL, metricHelp).labelNames(labels). + register(); + metricMap.put(metricName, ERROR_REQUESTS_RECEIVED_DATA_SERVICE); + } } @@ -242,7 +269,8 @@ public void serviceDown(String serviceName, String serviceType) { enum SERVICE { PROXY, API, - INBOUND_ENDPOINT + INBOUND_ENDPOINT, + DATA_SERVICE } /** @@ -259,6 +287,8 @@ private void createBuckets(Map configs) { MetricConstants.API_LATENCY_BUCKETS); Object inboundEndpointConfigBuckets = configs.get(MetricConstants.METRIC_HANDLER + "." + MetricConstants.INBOUND_ENDPOINT_LATENCY_BUCKETS); + Object dataServiceConfigBuckets = configs.get(MetricConstants.METRIC_HANDLER + "." + + MetricConstants.DATA_SERVICE_LATENCY_BUCKETS); if (null != proxyConfigBuckets) { List list = Arrays.asList(proxyConfigBuckets); @@ -284,6 +314,14 @@ private void createBuckets(Map configs) { inboundEndpointLatencyBuckets[i] = (double) bucketList.get(i); } } + if (null != dataServiceConfigBuckets) { + List list = Arrays.asList(dataServiceConfigBuckets); + int size = list.size(); + List bucketList = (ArrayList) list.get(0); + for (int i = 0; i < size; i++) { + dataServiceLatencyBuckets[i] = (double) bucketList.get(i); + } + } } /** @@ -333,6 +371,22 @@ public void initializeInboundEndpointMetrics() { initializeInboundEndpointErrorMetrics(); } + /** + * Create data services related metrics. + */ + public void initializeDataServiceMetrics() { + String[] labels = {MetricConstants.SERVICE_NAME, MetricConstants.SERVICE_TYPE}; + + createMetrics("DATA_SERVICE", MetricConstants.COUNTER, + MetricConstants.DATA_SERVICE_REQUEST_COUNT_TOTAL, + "Total number of requests to a data service.", labels); + createMetrics("DATA_SERVICE", MetricConstants.HISTOGRAM, + MetricConstants.DATA_SERVICE_LATENCY_SECONDS, + "Latency of requests to a data service.", labels); + + initializeDataServiceErrorMetrics(); + } + /** * Create the metrics related to failed proxy services. */ @@ -362,6 +416,16 @@ public void initializeInboundEndpointErrorMetrics() { new String[]{MetricConstants.SERVICE_NAME, MetricConstants.SERVICE_TYPE}); } + /** + * Create the metrics related to failed dataservices. + */ + public void initializeDataServiceErrorMetrics() { + initErrorMetrics("DATA_SERVICE", MetricConstants.COUNTER, + MetricConstants.DATA_SERVICE_REQUEST_COUNT_ERROR_TOTAL, "Total number of error" + + " requests when receiving the message by an inbound endpoint.", + new String[]{MetricConstants.SERVICE_NAME, MetricConstants.SERVICE_TYPE}); + } + /** * Create the metrics related to server startup. */ diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricConstants.java b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricConstants.java index 28887e5dc0..d81b206f39 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricConstants.java +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricConstants.java @@ -24,6 +24,7 @@ public class MetricConstants { // Constants for Synapse artifacts public static final String INBOUND_ENDPOINT = "inbound-endpoint"; + public static final String DATA_SERVICE = "data-service"; public static final String HTTP_PORT = "http.nio.port"; public static final String JAVA_VERSION = "java.vm.specification.version"; @@ -52,6 +53,13 @@ public class MetricConstants { public static final String INBOUND_ENDPOINT_LATENCY_SECONDS = "wso2_integration_inbound_endpoint_latency_seconds"; + public static final String DATA_SERVICE_REQUEST_COUNT_TOTAL = + "wso2_integration_data_service_request_count_total"; + public static final String DATA_SERVICE_REQUEST_COUNT_ERROR_TOTAL = + "wso2_integration_data_service_request_count_error_total"; + public static final String DATA_SERVICE_LATENCY_SECONDS = + "wso2_integration_data_service_latency_seconds"; + public static final String SERVER_UP = "wso2_integration_server_up"; public static final String SERVICE_UP = "wso2_integration_service_up"; public static final String SERVER_VERSION = "wso2_integration_server_version"; @@ -60,10 +68,12 @@ public class MetricConstants { public static final String PROXY_LATENCY_BUCKETS = "proxy_latency_buckets"; public static final String API_LATENCY_BUCKETS = "api_latency_buckets"; public static final String INBOUND_ENDPOINT_LATENCY_BUCKETS = "inbound_endpoint_latency_buckets"; + public static final String DATA_SERVICE_LATENCY_BUCKETS = "data_service_latency_buckets"; public static final String PROXY_LATENCY_TIMER = "PROXY_LATENCY_TIMER"; public static final String API_LATENCY_TIMER = "API_LATENCY_TIMER"; public static final String INBOUND_ENDPOINT_LATENCY_TIMER = "INBOUND_ENDPOINT_LATENCY_TIMER"; + public static final String DATA_SERVICE_LATENCY_TIMER = "DATA_SERVICE_LATENCY_TIMER"; public static final String SERVER = "Server"; public static final String SERVICE = "Service"; @@ -84,4 +94,8 @@ public class MetricConstants { public static final String VERSION_LABEL = "version"; public static final String UPDATE_LEVEL_LABEL = "update_level"; + public static final String MESSAGE_DIRECTION_IN = "in"; + + public static final String MESSAGE_DIRECTION_OUT = "out"; + } diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricUtils.java b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricUtils.java new file mode 100644 index 0000000000..7f46f95aee --- /dev/null +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/src/main/java/org/wso2/micro/integrator/observability/util/MetricUtils.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.micro.integrator.observability.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.wso2.config.mapper.ConfigParser; +import org.wso2.micro.integrator.observability.metric.handler.MetricReporter; +import org.wso2.micro.integrator.observability.metric.handler.prometheus.reporter.PrometheusReporter; + +import java.util.Map; + +public class MetricUtils { + + private static Log log = LogFactory.getLog(MetricUtils.class); + private static final String METRIC_REPORTER = "metric_reporter"; + + private static MetricReporter metricReporter = null; + + public static MetricReporter getMetricReporter() { + if (metricReporter == null) { + metricReporter = generateMetricReporter(); + } + return metricReporter; + } + + /** + * Load the MetricReporter class from the deployment.toml file if a user has defined a MetricReporter. + * Use default PrometheusReporter if the user hasn't defined a MetricReporter or an error occurs + * during custom MetricReporter class invocation. + */ + + public static MetricReporter generateMetricReporter() { + Map configs = ConfigParser.getParsedConfigs(); + Object metricReporterClass = configs.get(MetricConstants.METRIC_HANDLER + "." + METRIC_REPORTER); + Class loadedMetricClass; + MetricReporter reporterInstance; + + if (metricReporterClass != null) { + try { + loadedMetricClass = Class.forName(metricReporterClass.toString()); + reporterInstance = (MetricReporter) loadedMetricClass.newInstance(); + if (log.isDebugEnabled()) { + log.debug("The class " + metricReporterClass + " loaded successfully"); + } + } catch (IllegalAccessException | ClassNotFoundException | InstantiationException e) { + log.error("Error in loading the class " + metricReporterClass.toString() + + " .Hence loading the default PrometheusReporter class ", e); + reporterInstance = loadDefaultPrometheusReporter(); + } + } else { + reporterInstance = loadDefaultPrometheusReporter(); + } + return reporterInstance; + } + + private static MetricReporter loadDefaultPrometheusReporter() { + MetricReporter reporterInstance = new PrometheusReporter(); + if (log.isDebugEnabled()) { + log.debug("The class org.wso2.micro.integrator.obsrvability.handler.metrics.publisher.prometheus." + + "reporter.PrometheusReporter was loaded successfully"); + } + return reporterInstance; + } +} diff --git a/components/org.wso2.micro.integrator.initializer/pom.xml b/components/org.wso2.micro.integrator.initializer/pom.xml index 56150bcc78..31ba2a367e 100755 --- a/components/org.wso2.micro.integrator.initializer/pom.xml +++ b/components/org.wso2.micro.integrator.initializer/pom.xml @@ -104,6 +104,10 @@ commons-io commons-io + + org.wso2.ei + org.wso2.micro.integrator.observability + diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java index 3dc9b0923e..a277e1264f 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusConstants.java @@ -117,6 +117,11 @@ public static final class RegistryStore { public static final String SUSPEND_PERSISTENCE = "suspend.mediation.persistence"; public static final String CONNECTOR_SECURE_VAULT_CONFIG_REPOSITORY = "/repository/components/secure-vault"; + //constant for prometheus API + public static final String ENABLE_PROMETHEUS_API_PROPERTY = "enablePrometheusApi"; + public static final String DISPATCH_PHASE_NAME = "Dispatch"; + public static final String MESSAGE_OUT_PHASE_NAME = "MessageOut"; + public static final String ESB_DEBUG_SYSTEM_PROPERTY = "esb.debug"; public static final String ESB_DEBUG_EVENT_PORT = "synapse.debugger.port.event"; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java index 9b8bb3adc8..c5c295e77b 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java @@ -22,6 +22,7 @@ import org.apache.axis2.description.AxisServiceGroup; import org.apache.axis2.description.Parameter; import org.apache.axis2.engine.AxisConfiguration; +import org.apache.axis2.engine.Phase; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -71,6 +72,8 @@ import org.wso2.micro.integrator.initializer.utils.SynapseArtifactInitUtils; import org.wso2.micro.integrator.ndatasource.core.DataSourceService; import org.wso2.micro.integrator.ntask.core.service.TaskService; +import org.wso2.micro.integrator.observability.metric.handler.DSMetricHandler; +import org.wso2.micro.integrator.observability.metric.handler.MetricHandler; import org.wso2.securevault.SecurityConstants; import java.io.File; @@ -176,6 +179,33 @@ protected void activate(ComponentContext ctxt) { } SynapseEnvironment synapseEnvironment = contextInfo.getSynapseEnvironment(); List handlers = synapseEnvironment.getSynapseHandlers(); + if (System.getProperty(ServiceBusConstants.ENABLE_PROMETHEUS_API_PROPERTY) != null) { + if (!handlers.stream().anyMatch(c -> c instanceof MetricHandler)) { + handlers.add(new MetricHandler()); + } + AxisConfiguration axisConfig = configCtxSvc.getServerConfigContext().getAxisConfiguration(); + for (Phase inPhase : axisConfig.getInFlowPhases()) { + if (ServiceBusConstants.DISPATCH_PHASE_NAME.equals(inPhase.getPhaseName())) { + if (!inPhase.getHandlers().stream().anyMatch(c -> c instanceof DSMetricHandler)) { + inPhase.addHandler(new DSMetricHandler()); + } + } + } + for (Phase outPhase : axisConfig.getOutFlowPhases()) { + if (ServiceBusConstants.MESSAGE_OUT_PHASE_NAME.equals(outPhase.getPhaseName())) { + if (!outPhase.getHandlers().stream().anyMatch(c -> c instanceof DSMetricHandler)) { + outPhase.addHandler(new DSMetricHandler()); + } + } + } + for (Phase faultPhase : axisConfig.getOutFaultFlowPhases()) { + if (ServiceBusConstants.MESSAGE_OUT_PHASE_NAME.equals(faultPhase.getPhaseName())) { + if (!faultPhase.getHandlers().stream().anyMatch(c -> c instanceof DSMetricHandler)) { + faultPhase.addHandler(new DSMetricHandler()); + } + } + } + } Iterator iterator = handlers.iterator(); while (iterator.hasNext()) { SynapseHandler handler = iterator.next(); diff --git a/pom.xml b/pom.xml index f5bff574e1..5004208cd6 100644 --- a/pom.xml +++ b/pom.xml @@ -1450,6 +1450,11 @@ opencensus ${opencensus.orbit.version} + + org.wso2.ei + org.wso2.micro.integrator.observability + ${project.version} + From d0aadad41036770dcdb8f1a262de70d071aa2795 Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Thu, 10 Aug 2023 09:31:04 +0530 Subject: [PATCH 05/47] Enable configuring service-catalog version Add a system property to configure the service catalog version. This enables MI to connect different APIM versions other than its parallel version. Fixes wso2/api-manager/issues/2084 --- .../initializer/utils/Constants.java | 9 +++---- .../utils/ServiceCatalogUtils.java | 24 ++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java index 8f40854531..3a22eea641 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java @@ -42,10 +42,11 @@ public class Constants { public static final String USER_NAME = "username"; public static final String PASSWORD = "password"; public static final String SERVICE_CATALOG_EXECUTOR_THREADS = "executor_threads"; - - public static final String SERVICE_CATALOG_PUBLISH_ENDPOINT = "api/am/service-catalog/v1/services/import?overwrite" + - "=true"; - public static final String SERVICE_CATALOG_GET_SERVICES_ENDPOINT = "api/am/service-catalog/v1/services"; + public static final String SERVICE_CATALOG_API_VERSION_PROPERTY = "service-catalog.api.version"; + public static final String SERVICE_CATALOG_DEFAULT_API_VERSION = "v1"; + public static final String SERVICE_CATALOG_ENDPOINT_PREFIX = "api/am/service-catalog/"; + public static final String SERVICE_CATALOG_PUBLISH_ENDPOINT = "/services/import?overwrite=true"; + public static final String SERVICE_CATALOG_GET_SERVICES_ENDPOINT = "/services"; // creating the payload.zip related constants public static final String SERVICE_CATALOG = "ServiceCatalog"; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java index 295e8748ab..1a49bb3731 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java @@ -36,7 +36,6 @@ import org.wso2.micro.application.deployer.CarbonApplication; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.core.util.StringUtils; -import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; import org.wso2.micro.integrator.initializer.deployment.application.deployer.CappDeployer; import org.wso2.securevault.SecretResolver; import org.wso2.securevault.SecretResolverFactory; @@ -83,6 +82,15 @@ public class ServiceCatalogUtils { private static String resolvedUrl; private static String lineSeparator; private static Map parsedConfigs; + private static final String API_VERSION; + + static { + String apiVersion = System.getProperty(SERVICE_CATALOG_API_VERSION_PROPERTY); + if (apiVersion == null) { + apiVersion = SERVICE_CATALOG_DEFAULT_API_VERSION; + } + API_VERSION = apiVersion; + } /** * Update the service url by injecting env variables. @@ -241,11 +249,10 @@ public static Map getAllServices(Map apimConfigs (apimConfigs.get(USER_NAME) + ":" + apimConfigs.get(PASSWORD)).getBytes()); // create get all services url - if (APIMHost.endsWith("/")) { - APIMHost = APIMHost + SERVICE_CATALOG_GET_SERVICES_ENDPOINT; - } else { - APIMHost = APIMHost + "/" + SERVICE_CATALOG_GET_SERVICES_ENDPOINT; + if (!APIMHost.endsWith("/")) { + APIMHost = APIMHost + "/"; } + APIMHost = APIMHost + SERVICE_CATALOG_ENDPOINT_PREFIX + API_VERSION + SERVICE_CATALOG_GET_SERVICES_ENDPOINT; try { HttpsURLConnection connection = (HttpsURLConnection) new URL(APIMHost).openConnection(); @@ -303,11 +310,10 @@ private static int uploadZip(Map apimConfigs, String attachmentF String APIMHost = apimConfigs.get(APIM_HOST); // create POST URL - if (APIMHost.endsWith("/")) { - APIMHost = APIMHost + SERVICE_CATALOG_PUBLISH_ENDPOINT; - } else { - APIMHost = APIMHost + "/" + SERVICE_CATALOG_PUBLISH_ENDPOINT; + if (!APIMHost.endsWith("/")) { + APIMHost = APIMHost + "/"; } + APIMHost = APIMHost + SERVICE_CATALOG_ENDPOINT_PREFIX + API_VERSION + SERVICE_CATALOG_PUBLISH_ENDPOINT; String encodeBytes = Base64.getEncoder().encodeToString( From 9f1fe6408d5e50ecb276fd7f210647cd1c480316 Mon Sep 17 00:00:00 2001 From: Rosen Silva Date: Fri, 11 Aug 2023 12:23:18 +0530 Subject: [PATCH 06/47] Delete .github/ISSUE_TEMPLATE directory Remove the auto-redirection issues to api manager --- .github/ISSUE_TEMPLATE/config.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index b7163ad016..0000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -blank_issues_enabled: false -contact_links: -- name: '📚 Create an Issue' - about: Create an issue if something does not work as expected or suggest new functionality/improvements to the product suite. - url: https://github.com/wso2/api-manager/issues/new/choose -- name: '💬 Slack Channel' - url: 'https://apim-slack.wso2.com/' - about: | - Chat with the community to get quick clarifications for your questions. From 8d240f86b65bbe4e2ca47d247498e8ae966cd1ca Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Fri, 11 Aug 2023 16:26:47 +0530 Subject: [PATCH 07/47] Fix element order in data-service fault messages Fix element ordering in data-service fault messages to match the WSDL definition. Fixes wso2/product-ei/issues/4664 --- .../dataservices/core/DataServiceFault.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java index b3dd5318f0..09fb3155fe 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java @@ -18,6 +18,7 @@ package org.wso2.micro.integrator.dataservices.core; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -91,7 +92,19 @@ public static OMElement extractFaultMessage(Throwable throwable) { OMElement root = fac.createOMElement(new QName( DBConstants.WSO2_DS_NAMESPACE, DBConstants.DS_FAULT_ELEMENT)); OMNamespace ns = root.getNamespace(); - for (Map.Entry rootEntry : ((DataServiceFault) throwable).getPropertyMap().entrySet()) { + // Add values from hash map to a linked hash map to preserve the order + Map propLinkedHashMap = new LinkedHashMap<>(); + propLinkedHashMap.put(DBConstants.FaultParams.CURRENT_PARAMS, + ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.CURRENT_PARAMS)); + propLinkedHashMap.put(DBConstants.FaultParams.CURRENT_REQUEST_NAME, + ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.CURRENT_REQUEST_NAME)); + propLinkedHashMap.put(DBConstants.FaultParams.NESTED_EXCEPTION, + ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.NESTED_EXCEPTION)); + propLinkedHashMap.put(DBConstants.FaultParams.SOURCE_DATA_SERVICE, + ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.SOURCE_DATA_SERVICE)); + propLinkedHashMap.put(DBConstants.FaultParams.DS_CODE, + ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.DS_CODE)); + for (Map.Entry rootEntry : propLinkedHashMap.entrySet()) { OMElement keyElement = fac.createOMElement(rootEntry.getKey(), ns); if (rootEntry.getValue() instanceof Map) { for (Map.Entry dataServiceEntry : (Set) ((Map) rootEntry.getValue()).entrySet()) { From 02d298ba4f06958c367b1bb30e39f4d15e44ea7f Mon Sep 17 00:00:00 2001 From: sanoj Date: Wed, 16 Aug 2023 14:28:19 +0530 Subject: [PATCH 08/47] Add test case to verify JSON object names with special characters --- ...bjectNameWithSpecialCharacterTestCase.java | 74 +++++++++++++++++++ .../api/JsonObjectNameSpecialCharacter.xml | 36 +++++++++ 2 files changed, 110 insertions(+) create mode 100644 integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/json/JSONObjectNameWithSpecialCharacterTestCase.java create mode 100644 integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/JsonObjectNameSpecialCharacter.xml diff --git a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/json/JSONObjectNameWithSpecialCharacterTestCase.java b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/json/JSONObjectNameWithSpecialCharacterTestCase.java new file mode 100644 index 0000000000..5715210ce9 --- /dev/null +++ b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/json/JSONObjectNameWithSpecialCharacterTestCase.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.esb.json; + +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.wso2.esb.integration.common.utils.ESBIntegrationTest; +import org.wso2.esb.integration.common.utils.clients.SimpleHttpClient; + +/** + * This test case is written to verify the fix for https://github.com/wso2/api-manager/issues/2082 + * This test case will verify the JSON object name with special characters + */ +public class JSONObjectNameWithSpecialCharacterTestCase extends ESBIntegrationTest { + + private final SimpleHttpClient httpClient = new SimpleHttpClient(); + private final String expectedPayload = "{\n" + + " \"id\": \"001\",\n" + + " \"special_data\": {\n" + + " \"@odata.nextLink\": \"sample_url\",\n" + + " \"value\": [\n" + + " {\n" + + " \"id\": \"sample_id\",\n" + + " \"@removed\": {\n" + + " \"reason\": \"sample_reason\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + + @BeforeClass() + public void setEnvironment() throws Exception { + + super.init(); + checkApiExistence("JsonObjectNameSpecialCharacter"); + } + + @Test(groups = "wso2.esb", description = "Test JSONObject name with special character") + public void testJSONObjectNameWithSpecialCharacter() throws Exception { + + HttpResponse response = httpClient.doGet(getApiInvocationURL("JsonObjectNameSpecialCharacter"), null); + Assert.assertEquals(response.getStatusLine().getStatusCode(), 200); + // Assert the JSON payloads + Assert.assertEquals(new Gson().fromJson(SimpleHttpClient.responseEntityBodyToString(response), Object.class), + new Gson().fromJson(expectedPayload, Object.class)); + } + + @AfterClass() + public void destroy() throws Exception { + + super.cleanup(); + } +} diff --git a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/JsonObjectNameSpecialCharacter.xml b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/JsonObjectNameSpecialCharacter.xml new file mode 100644 index 0000000000..4bf423b42f --- /dev/null +++ b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/JsonObjectNameSpecialCharacter.xml @@ -0,0 +1,36 @@ + + + + + + { + "@odata.nextLink": "sample_url", + "value": [ + { + "id": "sample_id", + "@removed": { + "reason": "sample_reason" + } + } + ] + } + + + + + + { + "id": "001", + "special_data": $1 + } + + + + + + + + + From cebc1ff9ee8847a5bc24a68e6b87ab9a0fb3a83d Mon Sep 17 00:00:00 2001 From: sanoj Date: Fri, 18 Aug 2023 08:52:18 +0530 Subject: [PATCH 09/47] Bump synapse version to 4.0.0-wso2v45 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f5bff574e1..4632ad9953 100644 --- a/pom.xml +++ b/pom.xml @@ -1545,7 +1545,7 @@ 2.3.0 2.3.1 - 4.0.0-wso2v40 + 4.0.0-wso2v45 [4.0.0, 4.0.1) 4.7.175 1.1.3 From a89f428f5077aba74ee6bc97437a133d4f5f4a0f Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Fri, 18 Aug 2023 14:26:20 +0530 Subject: [PATCH 10/47] Add test case for element ordering in DSS fault Add test case to verify the element order in a DSS fault message. Related to wso2/product-ei/issues/4664 --- .../dataservices/core/DataServiceFault.java | 94 ++++++++-------- .../fault/ErrorMessageVerifyTestCase.java | 102 ++++++++++++++++++ .../tests/src/test/resources/testng.xml | 6 ++ 3 files changed, 155 insertions(+), 47 deletions(-) create mode 100644 integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java index 09fb3155fe..e35a148c5c 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java @@ -71,7 +71,7 @@ public class DataServiceFault extends Exception { /** * This map contains all the properties related to data services fault message */ - private Map propertyMap = new HashMap(); + private Map propertyMap = new LinkedHashMap<>(); public DataServiceFault(Throwable nestedException, String code, String dsFaultMessage) { super(nestedException); @@ -92,19 +92,7 @@ public static OMElement extractFaultMessage(Throwable throwable) { OMElement root = fac.createOMElement(new QName( DBConstants.WSO2_DS_NAMESPACE, DBConstants.DS_FAULT_ELEMENT)); OMNamespace ns = root.getNamespace(); - // Add values from hash map to a linked hash map to preserve the order - Map propLinkedHashMap = new LinkedHashMap<>(); - propLinkedHashMap.put(DBConstants.FaultParams.CURRENT_PARAMS, - ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.CURRENT_PARAMS)); - propLinkedHashMap.put(DBConstants.FaultParams.CURRENT_REQUEST_NAME, - ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.CURRENT_REQUEST_NAME)); - propLinkedHashMap.put(DBConstants.FaultParams.NESTED_EXCEPTION, - ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.NESTED_EXCEPTION)); - propLinkedHashMap.put(DBConstants.FaultParams.SOURCE_DATA_SERVICE, - ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.SOURCE_DATA_SERVICE)); - propLinkedHashMap.put(DBConstants.FaultParams.DS_CODE, - ((DataServiceFault) throwable).getPropertyMap().get(DBConstants.FaultParams.DS_CODE)); - for (Map.Entry rootEntry : propLinkedHashMap.entrySet()) { + for (Map.Entry rootEntry : ((DataServiceFault) throwable).getPropertyMap().entrySet()) { OMElement keyElement = fac.createOMElement(rootEntry.getKey(), ns); if (rootEntry.getValue() instanceof Map) { for (Map.Entry dataServiceEntry : (Set) ((Map) rootEntry.getValue()).entrySet()) { @@ -116,9 +104,11 @@ public static OMElement extractFaultMessage(Throwable throwable) { keyElement.addChild(dataServiceKeyElement); } } else { - OMText valueElement = fac.createOMText( - keyElement, rootEntry.getValue().toString()); - keyElement.addChild(valueElement); + if (rootEntry.getValue() != null) { + OMText valueElement = fac.createOMText( + keyElement, rootEntry.getValue().toString()); + keyElement.addChild(valueElement); + } } root.addChild(keyElement); } @@ -186,40 +176,50 @@ public String getMessage() { /** * Returns a detailed description of the data service fault. */ - public String getFullMessage() { - StringBuffer buff = new StringBuffer(); - if (this.getDsFaultMessage() != null) { - buff.append("DS Fault Message: " + this.getDsFaultMessage() + "\n"); - } - if (this.getCode() != null) { - buff.append("DS Code: " + this.getCode() + "\n"); - getPropertyMap().put(DBConstants.FaultParams.DS_CODE, this.getCode()); - } - if (this.getSourceDataService() != null) { - buff.append("Source Data Service:-\n"); - buff.append(this.getSourceDataService().toString()); - Map sourcePropertyMap = new HashMap(); - sourcePropertyMap.put(DBConstants.FaultParams.DATA_SERVICE_NAME, this.getSourceDataService().getName()); + public String getFullMessage() { + StringBuffer buff = new StringBuffer(); + if (this.getDsFaultMessage() != null) { + buff.append("DS Fault Message: " + this.getDsFaultMessage() + "\n"); + } + if (this.getCurrentParams() != null && !("true".equalsIgnoreCase(DBUtils.getCurrentParamsDisabledProperty()))) { + buff.append("Current Params: " + this.getCurrentParams() + "\n"); + getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, this.getCurrentParams().toString()); + } else { + getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, null); + } + if (this.getCurrentRequestName() != null) { + buff.append("Current Request Name: " + this.getCurrentRequestName() + "\n"); + getPropertyMap().put(DBConstants.FaultParams.CURRENT_REQUEST_NAME, this.getCurrentRequestName()); + } else { + getPropertyMap().put(DBConstants.FaultParams.CURRENT_REQUEST_NAME, null); + } + if (this.getCause() != null) { + buff.append("Nested Exception:-\n" + this.getCause() + "\n"); + getPropertyMap().put(DBConstants.FaultParams.NESTED_EXCEPTION, this.getCause().toString()); + } else { + getPropertyMap().put(DBConstants.FaultParams.NESTED_EXCEPTION, null); + } + if (this.getSourceDataService() != null) { + buff.append("Source Data Service:-\n"); + buff.append(this.getSourceDataService().toString()); + Map sourcePropertyMap = new LinkedHashMap<>(); sourcePropertyMap.put(DBConstants.FaultParams.LOCATION, this.getSourceDataService().getRelativeDsLocation()); + sourcePropertyMap.put(DBConstants.FaultParams.DEFAULT_NAMESPACE, this.getSourceDataService().getDefaultNamespace()); sourcePropertyMap.put(DBConstants.FaultParams.DESCRIPTION, this.getSourceDataService().getDescription() != null ? this.getSourceDataService().getDescription() : "N/A"); - sourcePropertyMap.put(DBConstants.FaultParams.DEFAULT_NAMESPACE, this.getSourceDataService().getDefaultNamespace()); + sourcePropertyMap.put(DBConstants.FaultParams.DATA_SERVICE_NAME, this.getSourceDataService().getName()); getPropertyMap().put(DBConstants.FaultParams.SOURCE_DATA_SERVICE, sourcePropertyMap); - } - if (this.getCurrentRequestName() != null) { - buff.append("Current Request Name: " + this.getCurrentRequestName() + "\n"); - getPropertyMap().put(DBConstants.FaultParams.CURRENT_REQUEST_NAME, this.getCurrentRequestName()); - } - if (this.getCurrentParams() != null && !("true".equalsIgnoreCase(DBUtils.getCurrentParamsDisabledProperty()))) { - buff.append("Current Params: " + this.getCurrentParams() + "\n"); - getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, this.getCurrentParams().toString()); - } - if (this.getCause() != null) { - buff.append("Nested Exception:-\n" + this.getCause() + "\n"); - getPropertyMap().put(DBConstants.FaultParams.NESTED_EXCEPTION, this.getCause().toString()); - } - return buff.toString(); - } + } else { + getPropertyMap().put(DBConstants.FaultParams.SOURCE_DATA_SERVICE, null); + } + if (this.getCode() != null) { + buff.append("DS Code: " + this.getCode() + "\n"); + getPropertyMap().put(DBConstants.FaultParams.DS_CODE, this.getCode()); + } else { + getPropertyMap().put(DBConstants.FaultParams.DS_CODE, null); + } + return buff.toString(); + } @Override public String toString() { diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java new file mode 100644 index 0000000000..ddd32111ba --- /dev/null +++ b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.ei.dataservice.integration.test.fault; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.wso2.ei.dataservice.integration.test.DSSIntegrationTest; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +import static org.testng.Assert.assertTrue; + +/** + * Test case to verify the element ordering of a DSS soap fault. + */ +public class ErrorMessageVerifyTestCase extends DSSIntegrationTest { + + @BeforeClass(alwaysRun = true) + public void serviceDeployment() throws Exception { + super.init(); + } + + @AfterClass(alwaysRun = true) + public void destroy() throws Exception { + cleanup(); + } + + @Test(groups = {"wso2.dss"}, description = "Check element ordering of a DSS soap fault") + public void verifyDSSErrorMessageTestCase() throws Exception { + String serviceEndPoint = "http://localhost:8480/services/ResourcesServiceTest/product"; + + // cannot use HttpRequestUtil.doPOST() because it fails when response is 500 + URL urlObj = new URL(serviceEndPoint); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/xml"); + connection.setRequestProperty("Accept", "application/xml"); + + byte[] postDataBytes = "invalid".getBytes(StandardCharsets.UTF_8); + try (DataOutputStream outputStream = new DataOutputStream(connection.getOutputStream())) { + outputStream.write(postDataBytes); + } + int responseCode = connection.getResponseCode(); + Assert.assertEquals(responseCode, 500, "Response code mismatched"); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()))) { + String line; + StringBuilder response = new StringBuilder(); + while ((line = reader.readLine()) != null) { + response.append(line); + } + int currentParamsIndex = response.indexOf(""); + int requestNameIndex = response.indexOf(""); + int nestedExceptionIndex = response.indexOf(""); + int locationIndex = response.indexOf(""); + int namespaceIndex = response.indexOf(""); + int descriptionIndex = response.indexOf(""); + int serviceNameIndex = response.indexOf(""); + int dsCodeIndex = response.indexOf(""); + assertTrue(currentParamsIndex < requestNameIndex, + "current_params should be before current_request_name"); + assertTrue(requestNameIndex < nestedExceptionIndex, + "current_request_name should be before nested_exception"); + assertTrue(nestedExceptionIndex < dataServiceIndex, + "nested_exception should be before source_data_service"); + assertTrue(dataServiceIndex < locationIndex, + "source_data_service should be before location"); + assertTrue(locationIndex < namespaceIndex, + "location should be before default_namespace"); + assertTrue(namespaceIndex < descriptionIndex, + "default_namespace should be before description"); + assertTrue(descriptionIndex < serviceNameIndex, + "description should be before data_service_name"); + assertTrue(serviceNameIndex < dsCodeIndex, + "data_service_name should be before ds_code"); + } + } +} diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/resources/testng.xml b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/resources/testng.xml index fbed17de05..53beb5c352 100755 --- a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/resources/testng.xml +++ b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/resources/testng.xml @@ -83,6 +83,12 @@ + + + + + + {{server.service_path}} + + {{server.strict_service_path_validation}} + {{server.services_directory}} diff --git a/pom.xml b/pom.xml index 4632ad9953..8205d4ae72 100644 --- a/pom.xml +++ b/pom.xml @@ -1568,7 +1568,7 @@ 2.1 - 1.6.1-wso2v91 + 1.6.1-wso2v94 1.6.1-wso2v64 [1.6.1, 1.7.0) From e078297d1c35f35cdf915d8bc4a1d7a14b1973e5 Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 23 Aug 2023 16:55:17 +0530 Subject: [PATCH 16/47] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a1e8468f0..7e5dafcbb3 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Please follow the steps below to build WSO2 Micro Integrator from the source cod 2. Run the maven command `mvn clean install` from the root directory of the repository. 3. The generated Micro Integrator distribution can be found at `micro-integrator/distribution/target/wso2mi-.zip`. -Please note that the product can be build using only JDK 8 but the integration tests can be run in either JDK 8 or 11. +Please note that the product can be build using only JDK 11 but the integration tests can be run in either JDK 11 or 17. #### Building the Docker image From 6d2bad1dbdc0e48b1cb1aceec7c86c6d56e68e8f Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Thu, 24 Aug 2023 08:13:23 +0530 Subject: [PATCH 17/47] Fix test failures due to namespace differences Fix test failures due to namespace differences --- .../test/fault/ErrorMessageVerifyTestCase.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java index ddd32111ba..2183b26df5 100644 --- a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java +++ b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java @@ -72,15 +72,15 @@ public void verifyDSSErrorMessageTestCase() throws Exception { while ((line = reader.readLine()) != null) { response.append(line); } - int currentParamsIndex = response.indexOf(""); - int requestNameIndex = response.indexOf(""); - int nestedExceptionIndex = response.indexOf(""); - int locationIndex = response.indexOf(""); - int namespaceIndex = response.indexOf(""); - int descriptionIndex = response.indexOf(""); - int serviceNameIndex = response.indexOf(""); - int dsCodeIndex = response.indexOf(""); + int currentParamsIndex = response.indexOf(":current_params>"); + int requestNameIndex = response.indexOf(":current_request_name>"); + int nestedExceptionIndex = response.indexOf(":nested_exception>"); + int dataServiceIndex = response.indexOf(":source_data_service>"); + int locationIndex = response.indexOf(":location>"); + int namespaceIndex = response.indexOf(":default_namespace>"); + int descriptionIndex = response.indexOf(":description>"); + int serviceNameIndex = response.indexOf(":data_service_name>"); + int dsCodeIndex = response.indexOf(":ds_code>"); assertTrue(currentParamsIndex < requestNameIndex, "current_params should be before current_request_name"); assertTrue(requestNameIndex < nestedExceptionIndex, From 0fd23dec8a5fb4beb63b0d55b5477e71945f8139 Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Thu, 24 Aug 2023 16:45:17 +0530 Subject: [PATCH 18/47] Fix typo Fix type in test case --- .../integration/test/fault/ErrorMessageVerifyTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java index 2183b26df5..3c469160e6 100644 --- a/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java +++ b/integration/dataservice-hosting-tests/tests-integration/tests/src/test/java/org/wso2/ei/dataservice/integration/test/fault/ErrorMessageVerifyTestCase.java @@ -74,7 +74,7 @@ public void verifyDSSErrorMessageTestCase() throws Exception { } int currentParamsIndex = response.indexOf(":current_params>"); int requestNameIndex = response.indexOf(":current_request_name>"); - int nestedExceptionIndex = response.indexOf(":nested_exception>"); + int nestedExceptionIndex = response.indexOf(":nested_exception"); int dataServiceIndex = response.indexOf(":source_data_service>"); int locationIndex = response.indexOf(":location>"); int namespaceIndex = response.indexOf(":default_namespace>"); From 46f1edac5de074888b568c0895c609ac3784f6a5 Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Tue, 29 Aug 2023 14:27:39 +0530 Subject: [PATCH 19/47] Fix issue in DSS error message. When current params is disabled, its getting removed from WSDL also. So, no need to add null element when its disabled. --- .../dataservices/core/DataServiceFault.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java index e35a148c5c..762ad951dd 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/src/main/java/org/wso2/micro/integrator/dataservices/core/DataServiceFault.java @@ -181,11 +181,15 @@ public String getFullMessage() { if (this.getDsFaultMessage() != null) { buff.append("DS Fault Message: " + this.getDsFaultMessage() + "\n"); } - if (this.getCurrentParams() != null && !("true".equalsIgnoreCase(DBUtils.getCurrentParamsDisabledProperty()))) { - buff.append("Current Params: " + this.getCurrentParams() + "\n"); - getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, this.getCurrentParams().toString()); - } else { - getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, null); + // if skipCurrentParams is not set to true, we don't have current params in WSDL + // so no need to add a null value + if (!("true".equalsIgnoreCase(DBUtils.getCurrentParamsDisabledProperty()))) { + if (this.getCurrentParams() != null) { + buff.append("Current Params: " + this.getCurrentParams() + "\n"); + getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, this.getCurrentParams().toString()); + } else { + getPropertyMap().put(DBConstants.FaultParams.CURRENT_PARAMS, null); + } } if (this.getCurrentRequestName() != null) { buff.append("Current Request Name: " + this.getCurrentRequestName() + "\n"); From e96aafde100f91c72d4496570025a161bf0c7485 Mon Sep 17 00:00:00 2001 From: malakaganga Date: Tue, 29 Aug 2023 11:55:43 +0530 Subject: [PATCH 20/47] Fix name extraction failure Fixes: https://github.com/wso2/micro-integrator/issues/2955 --- .../integrator/initializer/utils/SynapseArtifactInitUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/SynapseArtifactInitUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/SynapseArtifactInitUtils.java index abefdef5b8..e241d6bbf4 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/SynapseArtifactInitUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/SynapseArtifactInitUtils.java @@ -114,7 +114,7 @@ public boolean accept(File dir, String name) { String packageName = retrievePackageName(connectorExtractedPath); // Retrieve connector name - String connectorName = connectorZip.getName().substring(0, connectorZip.getName().indexOf('-')); + String connectorName = connectorZip.getName().substring(0, connectorZip.getName().lastIndexOf('-')); QName qualifiedName = new QName(packageName, connectorName); File importFile = new File(importsDir, qualifiedName.toString() + ".xml"); From 188d8e34b15804358ea90e308ad1695bb7147f26 Mon Sep 17 00:00:00 2001 From: Arunan Date: Tue, 5 Sep 2023 16:47:23 +0530 Subject: [PATCH 21/47] Stop transport listeners asynchronously during server shutdown --- .../org/wso2/micro/core/ServerManagement.java | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/core/ServerManagement.java b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/core/ServerManagement.java index 4517c378b9..ff6ada5ce4 100644 --- a/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/core/ServerManagement.java +++ b/components/org.wso2.micro.integrator.core/src/main/java/org/wso2/micro/core/ServerManagement.java @@ -27,9 +27,15 @@ import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; import java.lang.management.ManagementPermission; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import javax.management.MBeanServer; import javax.management.ObjectName; import javax.management.Query; @@ -73,13 +79,44 @@ public void startMaintenance() throws Exception { secMan.checkPermission(new ManagementPermission("control")); } log.info("Starting to switch to maintenance mode..."); + stopTransportListeners(); + destroyTransportListeners(); + waitForRequestCompletion(); + } + + /** + * Stop Transport Listeners asynchronously and wait for the completion of the tasks + */ + private void stopTransportListeners() { + ExecutorService transportListenerShutdownPool = Executors.newFixedThreadPool(inTransports.size()); + List> listenerShutdownFutures = new ArrayList<>(); for (TransportInDescription tinDesc : inTransports.values()) { TransportListener transport = tinDesc.getReceiver(); - transport.stop(); + Future future = transportListenerShutdownPool.submit(new TransportListenerShutdownTask(transport)); + listenerShutdownFutures.add(future); + } + + // Wait until shutting down the transport listeners before proceeding + for (Future future : listenerShutdownFutures) { + try { + future.get(); + } catch (Exception e) { + log.error("Error while completing transport listener shutdown", e); + } } + transportListenerShutdownPool.shutdown(); log.info("Stopped all transport listeners"); + } - waitForRequestCompletion(); + /** + * Destroy Transport Listeners + */ + private void destroyTransportListeners() { + // Destroy the TransportListener at the end to clear up resources + for (TransportInDescription tinDesc : inTransports.values()) { + TransportListener transport = tinDesc.getReceiver(); + transport.destroy(); + } } /** @@ -209,4 +246,24 @@ public void endMaintenance() throws Exception { } log.info("Switched to normal mode"); } + + /** + * Callable task to pause and shutdown a transport listener + */ + private class TransportListenerShutdownTask implements Callable { + private TransportListener transport; + + public TransportListenerShutdownTask(TransportListener transport) { + this.transport = transport; + } + + public Void call() throws Exception { + try { + transport.stop(); + } catch (Exception e) { + log.error("Error while stopping Transport Listener", e); + } + return null; + } + } } From 66da899c2db84f35481e6d5e3c64378de0b411f5 Mon Sep 17 00:00:00 2001 From: chanikag Date: Wed, 6 Sep 2023 09:56:56 +0530 Subject: [PATCH 22/47] Upgrade synapse version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f5bff574e1..6d56f97fcb 100644 --- a/pom.xml +++ b/pom.xml @@ -1545,7 +1545,7 @@ 2.3.0 2.3.1 - 4.0.0-wso2v40 + 4.0.0-wso2v47 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 8ecf3a2e1636ad806a07f6dee8b43fd0ad9f658e Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Fri, 23 Jun 2023 09:29:43 +0530 Subject: [PATCH 23/47] Implement the PoC for CDC inbound endpoint. Modify the payload removing unwanted information Add Securevault support for password encryption --- .../pom.xml | 20 ++ .../InboundRequestProcessorFactoryImpl.java | 5 +- .../protocol/cdc/CDCEndpointManager.java | 261 ++++++++++++++++++ .../protocol/cdc/CDCEventExecutorManager.java | 50 ++++ .../endpoint/protocol/cdc/CDCEventOutput.java | 142 ++++++++++ .../endpoint/protocol/cdc/CDCListener.java | 50 ++++ .../protocol/cdc/CDCSourceHandler.java | 165 +++++++++++ .../protocol/cdc/InboundCDCConstants.java | 51 ++++ .../protocol/cdc/InboundCDCEventExecutor.java | 49 ++++ .../src/main/resources/log4j.properties | 8 + .../CDCInbound/cdc-inbound-endpoint.xml | 19 ++ .../resources/CDCInbound/cdc_process_seq.xml | 16 ++ pom.xml | 107 ++++++- 13 files changed, 940 insertions(+), 3 deletions(-) create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml index c8f9d0d93d..bb7dd45d96 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml @@ -180,6 +180,26 @@ log4j-api test + + org.wso2.carbon.mediation + org.wso2.carbon.inbound.endpoint.persistence + + + io.debezium + debezium-embedded + + + io.debezium + debezium-api + + + io.debezium + debezium-connector-mysql + + + org.slf4j + slf4j-api + diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java index 389117f872..c8b4fb23db 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java @@ -21,6 +21,7 @@ import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundRequestProcessor; import org.apache.synapse.inbound.InboundRequestProcessorFactory; +import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedConsumer; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericInboundListener; @@ -44,7 +45,7 @@ */ public class InboundRequestProcessorFactoryImpl implements InboundRequestProcessorFactory { - public static enum Protocols {jms, file, http, https, hl7, kafka, mqtt, rabbitmq, ws, wss, grpc, httpws, httpswss} + public static enum Protocols {jms, file, http, https, hl7, kafka, mqtt, rabbitmq, ws, wss, grpc, httpws, httpswss, cdc} /** * return underlying Request Processor Implementation according to protocol @@ -97,6 +98,8 @@ public InboundRequestProcessor createInboundProcessor(InboundProcessorParams par case grpc: inboundRequestProcessor = new InboundGRPCListener(params); break; + case cdc: + inboundRequestProcessor = new CDCListener(params); } } else if (params.getClassImpl() != null) { if (GenericInboundListener.isListeningInboundEndpoint(params)) { diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java new file mode 100644 index 0000000000..a2320804ad --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java @@ -0,0 +1,261 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; +import org.apache.synapse.mediators.Value; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseException; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.util.xpath.SynapseXPath; +import org.jaxen.JaxenException; +import org.wso2.carbon.inbound.endpoint.common.AbstractInboundEndpointManager; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_CONNECTOR_CLASS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_DBNAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_HOSTNAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PORT; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_SERVER_ID; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_USER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OPERATIONS_EXCLUDE_LIST; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_MAX_THREADS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SNAPSHOT_MODE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TABLES_INCLUDE_LIST; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; + +public class CDCEndpointManager extends AbstractInboundEndpointManager { + + private static final Log log = LogFactory.getLog(CDCEndpointManager.class); + + private static CDCEndpointManager instance = null; + private InboundCDCEventExecutor eventExecutor; + + private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; + private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); + + private CDCEndpointManager() { + super(); + } + + public static CDCEndpointManager getInstance() { + if (instance == null) { + instance = new CDCEndpointManager(); + } + return instance; + } + + @Override + public boolean startListener(int port, String name, InboundProcessorParams inboundParameters) { + + if (CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { + log.info("CDC Listener already started on port " + port); + return true; + } + + log.info("Starting CDC Listener on port " + port); + + eventExecutor = new InboundCDCEventExecutor(); + CDCEventExecutorManager.getInstance().registerEventExecutor(port, eventExecutor); + Properties props = setProperties(inboundParameters); + + CDCSourceHandler sourceHandler = new CDCSourceHandler(port, inboundParameters); + DebeziumEngine> engine = DebeziumEngine.create(Json.class) + .using(props) + .notifying(record -> { + sourceHandler.requestReceived(record); + System.out.println(record); + }).build(); + + eventExecutor.getExecutorService().execute(engine); + log.info("Debezium engine started"); + + return true; + } + + private Properties setProperties (InboundProcessorParams params) { + Properties inboundProperties = params.getProperties(); + log.info("Initializing the properties"); + final Properties props = new Properties(); + try { + props.setProperty(DEBEZIUM_NAME, inboundProperties.getProperty(DEBEZIUM_NAME)); + if (inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE) != null) { + props.setProperty(DEBEZIUM_SNAPSHOT_MODE, inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE)); + } + + if (inboundProperties.getProperty(DEBEZIUM_MAX_THREADS) != null) { + props.setProperty(DEBEZIUM_MAX_THREADS, inboundProperties.getProperty(DEBEZIUM_MAX_THREADS)); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) != null) { + props.setProperty(DEBEZIUM_OFFSET_STORAGE, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE)); + } else { + props.setProperty(DEBEZIUM_OFFSET_STORAGE, "org.apache.kafka.connect.storage.FileOffsetBackingStore"); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) != null) { + props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME)); + } else { + String filePath = "cdc/offsetStorage/" + params.getName() + "_.dat"; + createFile(filePath); + props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) != null) { + props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS)); + } else { + props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); + } + + /* begin connector properties */ + props.setProperty(DEBEZIUM_CONNECTOR_CLASS, inboundProperties.getProperty(DEBEZIUM_CONNECTOR_CLASS)); + props.setProperty(DEBEZIUM_DATABASE_HOSTNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_HOSTNAME)); + props.setProperty(DEBEZIUM_DATABASE_PORT, inboundProperties.getProperty(DEBEZIUM_DATABASE_PORT)); + props.setProperty(DEBEZIUM_DATABASE_USER, inboundProperties.getProperty(DEBEZIUM_DATABASE_USER)); + + String passwordString = inboundProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); + SynapseEnvironment synapseEnvironment = params.getSynapseEnvironment(); + MessageContext messageContext = synapseEnvironment.createMessageContext(); + + props.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + + props.setProperty(DEBEZIUM_DATABASE_DBNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_DBNAME)); + props.setProperty(DEBEZIUM_DATABASE_SERVER_ID, inboundProperties.getProperty(DEBEZIUM_DATABASE_SERVER_ID)); + + if (inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) != null) { + props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL)); + } else { + props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); + } + + if (inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) != null) { + props.setProperty(DEBEZIUM_TOPIC_PREFIX, inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX)); + } else { + props.setProperty(DEBEZIUM_TOPIC_PREFIX, params.getName() +"_topic"); + } + + props.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + props.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + props.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + props.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); + + if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) != null) { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL)); + } else { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, "io.debezium.storage.file.history.FileSchemaHistory"); + } + + if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) != null) { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME)); + } else { + String filePath = "cdc/schemaHistory/" + params.getName() + "_.dat"; + createFile(filePath); + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); + } + + if (inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST) != null) { + props.setProperty(DEBEZIUM_TABLES_INCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST)); + } + + if (inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST) != null) { + props.setProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } catch (NullPointerException e) { + log.error("A required property value is not defined", e); + throw new RuntimeException(e); + } + return props; + } + + private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { + if (passwordString == null) { + return null; + } + Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); + String resolvedValue = ""; + if (lookupMatcher.find()) { + Value expression; + String expressionStr = lookupMatcher.group(1); + try { + expression = new Value(new SynapseXPath(expressionStr)); + + } catch (JaxenException e) { + throw new SynapseException("Error while building the expression : " + expressionStr, e); + } + resolvedValue = expression.evaluateValue(messageContext); + if (StringUtils.isEmpty(resolvedValue)) { + log.warn("Found Empty value for expression : " + expression.getExpression()); + resolvedValue = ""; + } + } else { + resolvedValue = passwordString; + } + return resolvedValue; + } + + @Override + public boolean startEndpoint(int port, String name, InboundProcessorParams inboundParameters) { + log.info("Starting CDC Endpoint on port " + port); + dataStore.registerListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME, InboundCDCConstants.CDC, name, + inboundParameters); + + boolean start = startListener(port, name, inboundParameters); + + if (start) { + //do nothing + } else { + dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); + return false; + } + return true; + } + + private void createFile (String filePath) throws IOException { + File file = new File(filePath); + file.getParentFile().mkdirs(); + if(!file.exists()) { + file.createNewFile(); + } + } + + @Override + public void closeEndpoint(int port) { + log.info("Closing CDC Endpoint on port " + port); + eventExecutor.getExecutorService().shutdown(); + dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); + + if (!CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { + log.info("Listener Endpoint is not started"); + return; + } else if (dataStore.isEndpointRegistryEmpty(port)) { + CDCEventExecutorManager.getInstance().shutdownExecutor(port); + } + + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java new file mode 100644 index 0000000000..80040b285f --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + + +import java.util.concurrent.ConcurrentHashMap; + +public class CDCEventExecutorManager { + + private ConcurrentHashMap executorPoolMap = new ConcurrentHashMap(); + + private static CDCEventExecutorManager instance = null; + + public static CDCEventExecutorManager getInstance() { + if (instance == null) { + instance = new CDCEventExecutorManager(); + } + return instance; + } + + public void shutdownExecutor(int port) { + executorPoolMap.get(port).shutdownEventExecutor(); + executorPoolMap.remove(port); + } + + public void registerEventExecutor(int port, InboundCDCEventExecutor eventExecutor) { + executorPoolMap.put(port, eventExecutor); + } + + public boolean isRegisteredExecutor(int port) { + return executorPoolMap.containsKey(port); + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java new file mode 100644 index 0000000000..df9439c101 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -0,0 +1,142 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.json.JSONObject; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.AFTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.BEFORE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DB; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OP; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.PAYLOAD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.SOURCE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; + + +public class CDCEventOutput { + + private Object beforeEvent; + private Object afterEvent; + private Long ts_ms; + + private String database; + private Object table; + private String op; + private JSONObject payload; + + private enum operations {c, r, u, d}; + + CDCEventOutput (ChangeEvent event) { + String valueString = event.value().toString(); + JSONObject value = new JSONObject(valueString); + this.payload = value.getJSONObject(PAYLOAD); + } + + public Object getJsonPayloadBeforeEvent() { + Object beforeObject = null; + if (payload.has(BEFORE)) { + beforeObject = payload.get(BEFORE); + } + return beforeObject; + + } + + public void setJsonPayloadBeforeEvent(Object beforeEvent) { + this.beforeEvent = beforeEvent; + } + + public Object getJsonPayloadAfterEvent() { + Object afterObject = null; + if (payload.has(AFTER)) { + afterObject = payload.get(AFTER); + } + return afterObject; + } + + public void setJsonPayloadAfterEvent(Object afterEvent) { + this.afterEvent = afterEvent; + } + + public Long getTs_ms() { + if (payload.has(TS_MS)) { + return payload.getLong(TS_MS); + } + return null; + } + + public void setTs_ms(Long ts_ms) { + this.ts_ms = ts_ms; + } + + public String getDatabase() { + if (getSource() != null) { + if (getSource().has(DB)) { + return getSource().getString(DB); + } + return null; + } + return null; + } + + public void setDatabase(String database) { + this.database = database; + } + + public Object getTable() { + Object tableObject = null; + if (getSource() != null) { + if (getSource().has(TABLE)) { + tableObject = getSource().get(TABLE); + } + } + return tableObject; + } + + public void setTable(Object table) { + this.table = table; + } + + private JSONObject getSource () { + if (payload.has(SOURCE)) { + return payload.getJSONObject(SOURCE); + } + return null; + } + + public String getOp() { + if (payload.has(OP)) { + return getOpString(payload.getString(OP)); + } + return null; + } + private String getOpString(String op) { + if (op != null) { + switch (operations.valueOf(op)) { + case c: + return "CREATE"; + case r: + return "READ"; + case u: + return "UPDATE"; + case d: + return "DELETE"; + } + } + return null; + } + + public void setOp(String op) { + this.op = op; + } + + public JSONObject getOutputJsonPayload () { + if (payload == null) { + return null; + } + JSONObject jsonPayload = new JSONObject(); + jsonPayload.put(OP, getOp()); + jsonPayload.put(BEFORE, getJsonPayloadBeforeEvent()); + jsonPayload.put(AFTER, getJsonPayloadAfterEvent()); + return jsonPayload; + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java new file mode 100644 index 0000000000..d249a37677 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java @@ -0,0 +1,50 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.SynapseException; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.inbound.InboundRequestProcessor; +import org.wso2.carbon.inbound.endpoint.persistence.PersistenceUtils; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.INBOUND_ENDPOINT_PARAMETER_CDC_PORT; + +public class CDCListener implements InboundRequestProcessor { + + private InboundProcessorParams processorParams; + private int port; + private String name; + private static final Log LOGGER = LogFactory.getLog(CDCListener.class); + + public CDCListener(InboundProcessorParams params) { + + processorParams = params; + String portParam = params.getProperties() + .getProperty(INBOUND_ENDPOINT_PARAMETER_CDC_PORT); + try { + port = Integer.parseInt(portParam); + } catch (NumberFormatException e) { + handleException("Validation failed for the port parameter " + portParam, e); + } + name = params.getName(); + } + + protected void handleException(String msg, Exception e) { + LOGGER.error(msg, e); + throw new SynapseException(msg, e); + } + + @Override + public void init() { + System.out.println("Init called"); + int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); + CDCEndpointManager.getInstance().startEndpoint(offsetPort, name, processorParams); + } + + @Override + public void destroy() { + System.out.println("Destroy called"); + int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); + CDCEndpointManager.getInstance().closeEndpoint(offsetPort); + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java new file mode 100644 index 0000000000..7df87f60ff --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java @@ -0,0 +1,165 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.axiom.om.OMAbstractFactory; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.soap.SOAPEnvelope; +import org.apache.axiom.soap.SOAPFactory; +import org.apache.axiom.util.UIDGenerator; +import org.apache.axis2.AxisFault; +import org.apache.axis2.builder.Builder; +import org.apache.axis2.builder.BuilderUtil; +import org.apache.axis2.context.OperationContext; +import org.apache.axis2.context.ServiceContext; +import org.apache.axis2.description.InOutAxisOperation; +import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.io.input.AutoCloseInputStream; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseConstants; +import org.apache.synapse.api.ApiConstants; +import org.apache.synapse.core.axis2.Axis2MessageContext; +import org.apache.synapse.core.axis2.MessageContextCreatorForAxis2; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.mediators.MediatorFaultHandler; +import org.apache.synapse.mediators.base.SequenceMediator; +import org.wso2.carbon.inbound.endpoint.osgi.service.ServiceReferenceHolder; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.IOException; + + +import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; +import static org.wso2.carbon.inbound.endpoint.common.Constants.TENANT_DOMAIN; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.*; + + +public class CDCSourceHandler { + + private int port; + private InboundProcessorParams params; + private static final String tenantDomain = SUPER_TENANT_DOMAIN_NAME; + private static final Log log = LogFactory.getLog(CDCSourceHandler.class); + private static final String contentType = "application/json"; + + public CDCSourceHandler(int port, InboundProcessorParams params) { + this.port = port; + this.params = params; + } + + public void requestReceived(ChangeEvent eventRecord) { + if (eventRecord == null || eventRecord.value() == null) { + log.debug("CDC Source Handler received empty event record"); + } else { + log.debug("CDC Source Handler request received"); + + MessageContext synCtx = null; + try { + + synCtx = getSynapseMessageContext(tenantDomain); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + synCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); + synCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); + synCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); + synCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + + org.apache.axis2.context.MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) synCtx) + .getAxis2MessageContext(); + Builder builder = BuilderUtil.getBuilderFromSelector(contentType, axis2MsgCtx); + + if (builder != null) { + String serializedChangeEvent = cdcEventOutput.getOutputJsonPayload().toString(); + InputStream in = new AutoCloseInputStream( + new ByteArrayInputStream(serializedChangeEvent.getBytes())); + + OMElement documentElement = builder.processDocument(in, contentType, axis2MsgCtx); + synCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); + + if (log.isDebugEnabled()) { + log.debug("CDCEvent being injected to Sequence"); + } + injectForMediation(synCtx); + return; + } + + } catch (AxisFault e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private static org.apache.axis2.context.MessageContext createAxis2MessageContext() { + org.apache.axis2.context.MessageContext axis2MsgCtx = new org.apache.axis2.context.MessageContext(); + axis2MsgCtx.setMessageID(UIDGenerator.generateURNString()); + axis2MsgCtx.setConfigurationContext( + ServiceReferenceHolder.getInstance().getConfigurationContextService().getServerConfigContext()); + axis2MsgCtx.setProperty(org.apache.axis2.context.MessageContext.CLIENT_API_NON_BLOCKING, Boolean.TRUE); + axis2MsgCtx.setServerSide(true); + + return axis2MsgCtx; + } + + private static org.apache.synapse.MessageContext createSynapseMessageContext(String tenantDomain) throws AxisFault { + org.apache.axis2.context.MessageContext axis2MsgCtx = createAxis2MessageContext(); + ServiceContext svcCtx = new ServiceContext(); + OperationContext opCtx = new OperationContext(new InOutAxisOperation(), svcCtx); + axis2MsgCtx.setServiceContext(svcCtx); + axis2MsgCtx.setOperationContext(opCtx); + + axis2MsgCtx.setProperty(TENANT_DOMAIN, tenantDomain); + + SOAPFactory fac = OMAbstractFactory.getSOAP11Factory(); + SOAPEnvelope envelope = fac.getDefaultEnvelope(); + axis2MsgCtx.setEnvelope(envelope); + return MessageContextCreatorForAxis2.getSynapseMessageContext(axis2MsgCtx); + } + + public org.apache.synapse.MessageContext getSynapseMessageContext(String tenantDomain) throws AxisFault { + MessageContext synCtx = createSynapseMessageContext(tenantDomain); + synCtx.setProperty(SynapseConstants.IS_INBOUND, true); + ((Axis2MessageContext) synCtx).getAxis2MessageContext().setProperty(SynapseConstants.IS_INBOUND, true); + + return synCtx; + } + + + private void injectForMediation(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = getFaultSequence(synCtx); + + MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); + synCtx.pushFaultHandler(mediatorFaultHandler); + if (log.isDebugEnabled()) { + log.debug("injecting message to sequence : " + params.getInjectingSeq()); + } + synCtx.setProperty("inbound.endpoint.name", params.getName()); + synCtx.setProperty(ApiConstants.API_CALLER, params.getName()); + + SequenceMediator injectingSequence = null; + if (params.getInjectingSeq() != null) { + injectingSequence = (SequenceMediator) synCtx.getSequence(params.getInjectingSeq()); + } + if (injectingSequence == null) { + injectingSequence = (SequenceMediator) synCtx.getMainSequence(); + } + + synCtx.getEnvironment().injectMessage(synCtx, injectingSequence); + + } + + private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = null; + if (params.getOnErrorSeq() != null) { + faultSequence = (SequenceMediator) synCtx.getSequence(params.getOnErrorSeq()); + } + if (faultSequence == null) { + faultSequence = (SequenceMediator) synCtx.getFaultSequence(); + } + return faultSequence; + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java new file mode 100644 index 0000000000..156a414e74 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -0,0 +1,51 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +public class InboundCDCConstants { + + /** Inbound Endpoint Parameters **/ + public static final String CDC = "cdc"; + public static final String INBOUND_ENDPOINT_PARAMETER_CDC_PORT = "inbound.cdc.port"; + + //debezium + public static final String DEBEZIUM_NAME = "name"; + public static final String DEBEZIUM_SNAPSHOT_MODE = "snapshot.mode"; + public static final String DEBEZIUM_MAX_THREADS = "snapshot.max.threads"; + public static final String DEBEZIUM_OFFSET_STORAGE = "offset.storage"; + public static final String DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME = "offset.storage.file.filename"; + public static final String DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS = "offset.flush.interval.ms"; + public static final String DEBEZIUM_CONNECTOR_CLASS = "connector.class"; + public static final String DEBEZIUM_DATABASE_HOSTNAME = "database.hostname"; + public static final String DEBEZIUM_DATABASE_PORT = "database.port"; + public static final String DEBEZIUM_DATABASE_USER = "database.user"; + public static final String DEBEZIUM_DATABASE_PASSWORD = "database.password"; + public static final String DEBEZIUM_DATABASE_DBNAME = "database.dbname"; + public static final String DEBEZIUM_TABLES_INCLUDE_LIST = "table.include.list"; + public static final String DEBEZIUM_OPERATIONS_EXCLUDE_LIST = "skipped.operations"; + public static final String DEBEZIUM_DATABASE_SERVER_ID = "database.server.id"; + public static final String DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL = "database.allowPublicKeyRetrieval"; + public static final String DEBEZIUM_TOPIC_PREFIX = "topic.prefix"; + + public static final String DEBEZIUM_VALUE_CONVERTER = "value.converter"; + public static final String DEBEZIUM_KEY_CONVERTER = "key.converter"; + public static final String DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE = "value.converter.schemas.enable"; + public static final String DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE = "key.converter.schemas.enable"; + + public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL = "schema.history.internal"; + public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME = "schema.history.internal.file.filename"; + + /** Output Properties **/ + public static final String DATABASE_NAME = "database"; + public static final String TABLES ="tables"; + public static final String OPERATIONS ="operations"; + public static final String TS_MS = "ts_ms"; + + public static final String BEFORE = "before"; + public static final String AFTER = "after"; + public static final String SOURCE = "source"; + public static final String OP = "op"; + public static final String PAYLOAD = "payload"; + public static final String DB = "db"; + public static final String TABLE = "table"; + + public static final String TRUE = "true"; +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java new file mode 100644 index 0000000000..37e3fbd70d --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class InboundCDCEventExecutor { + + private ExecutorService executorService; + + public InboundCDCEventExecutor () { + executorService = Executors.newSingleThreadExecutor(); + } + + public ExecutorService getExecutorService() { + return this.executorService; + } + + public void shutdownEventExecutor() { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(800, TimeUnit.MILLISECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + } + } + + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties new file mode 100644 index 0000000000..1da8230168 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml new file mode 100644 index 0000000000..e3caf7bc88 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml @@ -0,0 +1,19 @@ + + + 9091 + engine + org.apache.kafka.connect.storage.FileOffsetBackingStore + + io.debezium.connector.mysql.MySqlConnector + localhost + 3306 + root + rusiri@wso2 + students + 85744 + io.debezium.storage.file.history.FileSchemaHistory + + students.marks + c + + \ No newline at end of file diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml new file mode 100644 index 0000000000..16bab87ac4 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 6d56f97fcb..d184d4e4cf 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.wso2.ei wso2-micro-integrator-parent pom - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT WSO2 Micro Integrator http://wso2.com/products/enterprise-integrator/ WSO2 Micro Integrator @@ -1450,6 +1450,103 @@ opencensus ${opencensus.orbit.version} + + io.debezium + debezium-embedded + ${debezium.version} + + + io.debezium + debezium-api + ${debezium.version} + + + io.debezium + debezium-connector-mysql + ${debezium.version} + + + io.debezium + debezium-core + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-storage-kafka + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-storage-file + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-ddl-parser + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + com.github.shyiko + mysql-binlog-connector-java + 0.27.2 + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + org.slf4j + slf4j-api + + + + + org.apache.kafka + connect-runtime + ${kafka.version} + + + org.slf4j + slf4j-api + + + + + org.apache.kafka + connect-json + ${kafka.version} + + + org.slf4j + slf4j-api + + + @@ -1545,7 +1642,13 @@ 2.3.0 2.3.1 - 4.0.0-wso2v47 + + 2.4.1.Final + + + 3.3.1 + + 4.0.0-wso2v38 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 8b6fb96dc31d0e951ef5442f39bf25a429f27ac0 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Mon, 10 Jul 2023 15:58:32 +0530 Subject: [PATCH 24/47] Extend poc with Polling mode and clustering support Add dependencies for MSSQL, Oracle and Postgres --- .../InboundRequestProcessorFactoryImpl.java | 4 +- .../protocol/cdc/CDCEndpointManager.java | 261 ----------------- .../protocol/cdc/CDCEventExecutorManager.java | 50 ---- .../endpoint/protocol/cdc/CDCEventOutput.java | 53 ++-- .../protocol/cdc/CDCInjectHandler.java | 184 ++++++++++++ .../endpoint/protocol/cdc/CDCListener.java | 50 ---- .../protocol/cdc/CDCPollingConsumer.java | 118 ++++++++ .../endpoint/protocol/cdc/CDCProcessor.java | 265 ++++++++++++++++++ .../protocol/cdc/CDCSourceHandler.java | 165 ----------- .../endpoint/protocol/cdc/CDCTask.java | 67 +++++ .../protocol/cdc/InboundCDCConstants.java | 35 ++- .../protocol/cdc/InboundCDCEventExecutor.java | 49 ---- .../src/main/resources/log4j.properties | 8 - pom.xml | 24 +- 14 files changed, 699 insertions(+), 634 deletions(-) delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java index c8b4fb23db..77c4d16811 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java @@ -21,7 +21,7 @@ import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundRequestProcessor; import org.apache.synapse.inbound.InboundRequestProcessorFactory; -import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCListener; +import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedConsumer; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericInboundListener; @@ -99,7 +99,7 @@ public InboundRequestProcessor createInboundProcessor(InboundProcessorParams par inboundRequestProcessor = new InboundGRPCListener(params); break; case cdc: - inboundRequestProcessor = new CDCListener(params); + inboundRequestProcessor = new CDCProcessor(params); } } else if (params.getClassImpl() != null) { if (GenericInboundListener.isListeningInboundEndpoint(params)) { diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java deleted file mode 100644 index a2320804ad..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java +++ /dev/null @@ -1,261 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.format.Json; -import org.apache.synapse.mediators.Value; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; -import org.apache.synapse.SynapseException; -import org.apache.synapse.core.SynapseEnvironment; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.util.xpath.SynapseXPath; -import org.jaxen.JaxenException; -import org.wso2.carbon.inbound.endpoint.common.AbstractInboundEndpointManager; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_CONNECTOR_CLASS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_DBNAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_HOSTNAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PORT; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_SERVER_ID; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_USER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OPERATIONS_EXCLUDE_LIST; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_MAX_THREADS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SNAPSHOT_MODE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TABLES_INCLUDE_LIST; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; - -public class CDCEndpointManager extends AbstractInboundEndpointManager { - - private static final Log log = LogFactory.getLog(CDCEndpointManager.class); - - private static CDCEndpointManager instance = null; - private InboundCDCEventExecutor eventExecutor; - - private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; - private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); - - private CDCEndpointManager() { - super(); - } - - public static CDCEndpointManager getInstance() { - if (instance == null) { - instance = new CDCEndpointManager(); - } - return instance; - } - - @Override - public boolean startListener(int port, String name, InboundProcessorParams inboundParameters) { - - if (CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { - log.info("CDC Listener already started on port " + port); - return true; - } - - log.info("Starting CDC Listener on port " + port); - - eventExecutor = new InboundCDCEventExecutor(); - CDCEventExecutorManager.getInstance().registerEventExecutor(port, eventExecutor); - Properties props = setProperties(inboundParameters); - - CDCSourceHandler sourceHandler = new CDCSourceHandler(port, inboundParameters); - DebeziumEngine> engine = DebeziumEngine.create(Json.class) - .using(props) - .notifying(record -> { - sourceHandler.requestReceived(record); - System.out.println(record); - }).build(); - - eventExecutor.getExecutorService().execute(engine); - log.info("Debezium engine started"); - - return true; - } - - private Properties setProperties (InboundProcessorParams params) { - Properties inboundProperties = params.getProperties(); - log.info("Initializing the properties"); - final Properties props = new Properties(); - try { - props.setProperty(DEBEZIUM_NAME, inboundProperties.getProperty(DEBEZIUM_NAME)); - if (inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE) != null) { - props.setProperty(DEBEZIUM_SNAPSHOT_MODE, inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE)); - } - - if (inboundProperties.getProperty(DEBEZIUM_MAX_THREADS) != null) { - props.setProperty(DEBEZIUM_MAX_THREADS, inboundProperties.getProperty(DEBEZIUM_MAX_THREADS)); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) != null) { - props.setProperty(DEBEZIUM_OFFSET_STORAGE, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE)); - } else { - props.setProperty(DEBEZIUM_OFFSET_STORAGE, "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) != null) { - props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME)); - } else { - String filePath = "cdc/offsetStorage/" + params.getName() + "_.dat"; - createFile(filePath); - props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) != null) { - props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS)); - } else { - props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); - } - - /* begin connector properties */ - props.setProperty(DEBEZIUM_CONNECTOR_CLASS, inboundProperties.getProperty(DEBEZIUM_CONNECTOR_CLASS)); - props.setProperty(DEBEZIUM_DATABASE_HOSTNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_HOSTNAME)); - props.setProperty(DEBEZIUM_DATABASE_PORT, inboundProperties.getProperty(DEBEZIUM_DATABASE_PORT)); - props.setProperty(DEBEZIUM_DATABASE_USER, inboundProperties.getProperty(DEBEZIUM_DATABASE_USER)); - - String passwordString = inboundProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); - SynapseEnvironment synapseEnvironment = params.getSynapseEnvironment(); - MessageContext messageContext = synapseEnvironment.createMessageContext(); - - props.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); - - props.setProperty(DEBEZIUM_DATABASE_DBNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_DBNAME)); - props.setProperty(DEBEZIUM_DATABASE_SERVER_ID, inboundProperties.getProperty(DEBEZIUM_DATABASE_SERVER_ID)); - - if (inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) != null) { - props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL)); - } else { - props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); - } - - if (inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) != null) { - props.setProperty(DEBEZIUM_TOPIC_PREFIX, inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX)); - } else { - props.setProperty(DEBEZIUM_TOPIC_PREFIX, params.getName() +"_topic"); - } - - props.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - props.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - props.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); - props.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); - - if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) != null) { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL)); - } else { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, "io.debezium.storage.file.history.FileSchemaHistory"); - } - - if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) != null) { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME)); - } else { - String filePath = "cdc/schemaHistory/" + params.getName() + "_.dat"; - createFile(filePath); - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); - } - - if (inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST) != null) { - props.setProperty(DEBEZIUM_TABLES_INCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST)); - } - - if (inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST) != null) { - props.setProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST)); - } - } catch (IOException e) { - throw new RuntimeException(e); - } catch (NullPointerException e) { - log.error("A required property value is not defined", e); - throw new RuntimeException(e); - } - return props; - } - - private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { - if (passwordString == null) { - return null; - } - Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); - String resolvedValue = ""; - if (lookupMatcher.find()) { - Value expression; - String expressionStr = lookupMatcher.group(1); - try { - expression = new Value(new SynapseXPath(expressionStr)); - - } catch (JaxenException e) { - throw new SynapseException("Error while building the expression : " + expressionStr, e); - } - resolvedValue = expression.evaluateValue(messageContext); - if (StringUtils.isEmpty(resolvedValue)) { - log.warn("Found Empty value for expression : " + expression.getExpression()); - resolvedValue = ""; - } - } else { - resolvedValue = passwordString; - } - return resolvedValue; - } - - @Override - public boolean startEndpoint(int port, String name, InboundProcessorParams inboundParameters) { - log.info("Starting CDC Endpoint on port " + port); - dataStore.registerListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME, InboundCDCConstants.CDC, name, - inboundParameters); - - boolean start = startListener(port, name, inboundParameters); - - if (start) { - //do nothing - } else { - dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); - return false; - } - return true; - } - - private void createFile (String filePath) throws IOException { - File file = new File(filePath); - file.getParentFile().mkdirs(); - if(!file.exists()) { - file.createNewFile(); - } - } - - @Override - public void closeEndpoint(int port) { - log.info("Closing CDC Endpoint on port " + port); - eventExecutor.getExecutorService().shutdown(); - dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); - - if (!CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { - log.info("Listener Endpoint is not started"); - return; - } else if (dataStore.isEndpointRegistryEmpty(port)) { - CDCEventExecutorManager.getInstance().shutdownExecutor(port); - } - - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java deleted file mode 100644 index 80040b285f..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - - -import java.util.concurrent.ConcurrentHashMap; - -public class CDCEventExecutorManager { - - private ConcurrentHashMap executorPoolMap = new ConcurrentHashMap(); - - private static CDCEventExecutorManager instance = null; - - public static CDCEventExecutorManager getInstance() { - if (instance == null) { - instance = new CDCEventExecutorManager(); - } - return instance; - } - - public void shutdownExecutor(int port) { - executorPoolMap.get(port).shutdownEventExecutor(); - executorPoolMap.remove(port); - } - - public void registerEventExecutor(int port, InboundCDCEventExecutor eventExecutor) { - executorPoolMap.put(port, eventExecutor); - } - - public boolean isRegisteredExecutor(int port) { - return executorPoolMap.containsKey(port); - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index df9439c101..ec3af80f6a 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.wso2.carbon.inbound.endpoint.protocol.cdc; import io.debezium.engine.ChangeEvent; @@ -12,21 +30,13 @@ import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLE; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; - public class CDCEventOutput { - private Object beforeEvent; - private Object afterEvent; - private Long ts_ms; - - private String database; - private Object table; - private String op; private JSONObject payload; private enum operations {c, r, u, d}; - CDCEventOutput (ChangeEvent event) { + CDCEventOutput(ChangeEvent event) { String valueString = event.value().toString(); JSONObject value = new JSONObject(valueString); this.payload = value.getJSONObject(PAYLOAD); @@ -41,10 +51,6 @@ public Object getJsonPayloadBeforeEvent() { } - public void setJsonPayloadBeforeEvent(Object beforeEvent) { - this.beforeEvent = beforeEvent; - } - public Object getJsonPayloadAfterEvent() { Object afterObject = null; if (payload.has(AFTER)) { @@ -53,10 +59,6 @@ public Object getJsonPayloadAfterEvent() { return afterObject; } - public void setJsonPayloadAfterEvent(Object afterEvent) { - this.afterEvent = afterEvent; - } - public Long getTs_ms() { if (payload.has(TS_MS)) { return payload.getLong(TS_MS); @@ -64,10 +66,6 @@ public Long getTs_ms() { return null; } - public void setTs_ms(Long ts_ms) { - this.ts_ms = ts_ms; - } - public String getDatabase() { if (getSource() != null) { if (getSource().has(DB)) { @@ -78,10 +76,6 @@ public String getDatabase() { return null; } - public void setDatabase(String database) { - this.database = database; - } - public Object getTable() { Object tableObject = null; if (getSource() != null) { @@ -92,10 +86,6 @@ public Object getTable() { return tableObject; } - public void setTable(Object table) { - this.table = table; - } - private JSONObject getSource () { if (payload.has(SOURCE)) { return payload.getJSONObject(SOURCE); @@ -109,6 +99,7 @@ public String getOp() { } return null; } + private String getOpString(String op) { if (op != null) { switch (operations.valueOf(op)) { @@ -125,10 +116,6 @@ private String getOpString(String op) { return null; } - public void setOp(String op) { - this.op = op; - } - public JSONObject getOutputJsonPayload () { if (payload == null) { return null; diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java new file mode 100644 index 0000000000..00e5fe2127 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.util.UUIDGenerator; +import org.apache.axis2.AxisFault; +import org.apache.axis2.context.MessageContext; +import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.SynapseConstants; +import org.apache.synapse.SynapseException; +import org.apache.synapse.commons.json.JsonUtil; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundEndpoint; +import org.apache.synapse.mediators.MediatorFaultHandler; +import org.apache.synapse.mediators.base.SequenceMediator; +import org.apache.synapse.transport.customlogsetter.CustomLogSetter; +import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericConstants; + +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DATABASE_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OPERATIONS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLES; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; + +public class CDCInjectHandler { + + private static final Log logger = LogFactory.getLog(CDCInjectHandler.class); + + private String injectingSeq; + private String onErrorSeq; + private boolean sequential; + private Properties cdcProperties; + private SynapseEnvironment synapseEnvironment; + private Map transportHeaders; + private static final String contentType = "application/json"; + + public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequential, + SynapseEnvironment synapseEnvironment, Properties cdcProperties) { + this.injectingSeq = injectingSeq; + this.onErrorSeq = onErrorSeq; + this.sequential = sequential; + this.synapseEnvironment = synapseEnvironment; + this.cdcProperties = cdcProperties; + } + + /** + * Inject the message to the sequence + */ + public boolean invoke(Object object, String inboundEndpointName) throws SynapseException { + + ChangeEvent eventRecord = (ChangeEvent) object; + if (eventRecord == null || eventRecord.value() == null) { + if (logger.isDebugEnabled()) { + logger.debug("CDC Source Handler received empty event record"); + } + } else { + InputStream in = null; + try { + org.apache.synapse.MessageContext msgCtx = createMessageContext(); + msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); + msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); + msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); + InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); + CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + msgCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); + msgCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); + msgCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); + msgCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + + if (logger.isDebugEnabled()) { + logger.debug("Processed event : " + eventRecord); + } + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + + //Builder builder = null; + OMElement documentElement = null; + try { + documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, + cdcEventOutput.getOutputJsonPayload().toString(), true, true); + + } catch (AxisFault ex) { + logger.error("Error while creating the OMElement", ex); + msgCtx.setProperty(SynapseConstants.ERROR_CODE, GenericConstants.INBOUND_BUILD_ERROR); + msgCtx.setProperty(SynapseConstants.ERROR_MESSAGE, ex.getMessage()); + SequenceMediator faultSequence = getFaultSequence(msgCtx); + faultSequence.mediate(msgCtx); + return true; + } + + // Inject the message to the sequence. + msgCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); + if (injectingSeq == null || injectingSeq.equals("")) { + logger.error("Sequence name not specified. Sequence : " + injectingSeq); + return false; + } + SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() + .getSequence(injectingSeq); + if (seq != null) { + if (logger.isDebugEnabled()) { + logger.debug("injecting message to sequence : " + injectingSeq); + } + if (!seq.isInitialized()) { + seq.init(synapseEnvironment); + } + SequenceMediator faultSequence = getFaultSequence(msgCtx); + MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); + msgCtx.pushFaultHandler(mediatorFaultHandler); + + if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { + return false; + } + } else { + logger.error("Sequence: " + injectingSeq + " not found"); + } + + } catch (AxisFault e) { + throw new RuntimeException(e); + } + } + return true; + } + + private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = null; + if (this.onErrorSeq != null) { + faultSequence = (SequenceMediator) synCtx.getSequence(this.onErrorSeq); + } + + if (faultSequence == null) { + faultSequence = (SequenceMediator) synCtx.getFaultSequence(); + } + + return faultSequence; + } + + + /** + * @param transportHeaders the transportHeaders to set + */ + public void setTransportHeaders(Map transportHeaders) { + this.transportHeaders = transportHeaders; + } + + /** + * Create the initial message context for the file + */ + private org.apache.synapse.MessageContext createMessageContext() { + + org.apache.synapse.MessageContext msgCtx = synapseEnvironment.createMessageContext(); + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + axis2MsgCtx.setServerSide(true); + axis2MsgCtx.setMessageID(UUIDGenerator.getUUID()); + axis2MsgCtx.setProperty(MessageContext.TRANSPORT_HEADERS, transportHeaders); + msgCtx.setProperty(MessageContext.CLIENT_API_NON_BLOCKING, true); + return msgCtx; + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java deleted file mode 100644 index d249a37677..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.SynapseException; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.inbound.InboundRequestProcessor; -import org.wso2.carbon.inbound.endpoint.persistence.PersistenceUtils; - -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.INBOUND_ENDPOINT_PARAMETER_CDC_PORT; - -public class CDCListener implements InboundRequestProcessor { - - private InboundProcessorParams processorParams; - private int port; - private String name; - private static final Log LOGGER = LogFactory.getLog(CDCListener.class); - - public CDCListener(InboundProcessorParams params) { - - processorParams = params; - String portParam = params.getProperties() - .getProperty(INBOUND_ENDPOINT_PARAMETER_CDC_PORT); - try { - port = Integer.parseInt(portParam); - } catch (NumberFormatException e) { - handleException("Validation failed for the port parameter " + portParam, e); - } - name = params.getName(); - } - - protected void handleException(String msg, Exception e) { - LOGGER.error(msg, e); - throw new SynapseException(msg, e); - } - - @Override - public void init() { - System.out.println("Init called"); - int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); - CDCEndpointManager.getInstance().startEndpoint(offsetPort, name, processorParams); - } - - @Override - public void destroy() { - System.out.println("Destroy called"); - int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); - CDCEndpointManager.getInstance().closeEndpoint(offsetPort); - } -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java new file mode 100644 index 0000000000..dbd56d18f2 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.core.SynapseEnvironment; + +import java.util.Date; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor.inboundEpEventQueueMap; + +/** + * This class implement the processing logic related to inbound CDC protocol. + * Common functionalities are include in synapse + * util that is found in synapse commons + */ +public class CDCPollingConsumer { + + private static final Log logger = LogFactory.getLog(CDCPollingConsumer.class); + private Properties cdcProperties; + private String inboundEndpointName; + private SynapseEnvironment synapseEnvironment; + private long scanInterval; + private Long lastRanTime; + private CDCInjectHandler injectHandler; + + public CDCPollingConsumer(Properties cdcProperties, String inboundEndpointName, SynapseEnvironment synapseEnvironment, + long scanInterval) { + this.cdcProperties = cdcProperties; + this.inboundEndpointName = inboundEndpointName; + this.synapseEnvironment = synapseEnvironment; + this.scanInterval = scanInterval; + this.lastRanTime = null; + } + + /** + * Register a handler to process the file stream after reading from the + * source + * + * @param injectHandler + */ + public void registerHandler(CDCInjectHandler injectHandler) { + this.injectHandler = injectHandler; + } + + /** + * This will be called by the task scheduler. If a cycle execution takes + * more than the schedule interval, tasks will call this method ignoring the + * interval. Timestamp based check is done to avoid that. + */ + public void execute() { + try { + if (logger.isDebugEnabled()) { + logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); + } + // Check if the cycles are running in correct interval and start + // scan + long currentTime = (new Date()).getTime(); + if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { + lastRanTime = currentTime; + poll(); + } else if (logger.isDebugEnabled()) { + logger.debug( + "Skip cycle since cuncurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); + } + if (logger.isDebugEnabled()) { + logger.debug("End : CDC Inbound EP : " + inboundEndpointName); + } + } catch (Exception e) { + logger.error("Error while getting events. " + e.getMessage(), e); + } + } + + /** + * Do the CDC processing operation for the given set of properties. Then inject + * according to the registered handler + */ + public ChangeEvent poll() { + + if (logger.isDebugEnabled()) { + logger.debug("Start : listening to DB events : "); + } + + BlockingQueue> eventQueue = inboundEpEventQueueMap.get(inboundEndpointName); + while (!eventQueue.isEmpty()) { + injectHandler.invoke(eventQueue.poll(), inboundEndpointName); + } + + if (logger.isDebugEnabled()) { + logger.debug("End : Listening to DB events : "); + } + return null; + } + + protected Properties getInboundProperties() { + return cdcProperties; + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java new file mode 100644 index 0000000000..f5353e2acd --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseException; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.inbound.InboundTaskProcessor; +import org.apache.synapse.mediators.Value; +import org.apache.synapse.task.TaskStartupObserver; +import org.apache.synapse.util.xpath.SynapseXPath; +import org.jaxen.JaxenException; +import org.wso2.carbon.inbound.endpoint.common.InboundRequestProcessorImpl; +import org.wso2.carbon.inbound.endpoint.common.InboundTask; +import org.wso2.carbon.inbound.endpoint.protocol.PollingConstants; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; + +public class CDCProcessor extends InboundRequestProcessorImpl implements TaskStartupObserver, InboundTaskProcessor { + + private CDCPollingConsumer pollingConsumer; + private Properties cdcProperties; + private String injectingSeq; + private String onErrorSeq; + private boolean sequential; + + private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; + private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); + private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; + private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; + private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; + private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); + protected static Map inboundEpEventQueueMap = new HashMap(); + private ExecutorService executorService = null; + + public CDCProcessor(InboundProcessorParams params) { + this.name = params.getName(); + this.injectingSeq = params.getInjectingSeq(); + this.onErrorSeq = params.getOnErrorSeq(); + this.synapseEnvironment = params.getSynapseEnvironment(); + this.cdcProperties = params.getProperties(); + setProperties(); + try { + this.interval = Long.parseLong(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_INTERVAL)); + } catch (NumberFormatException nfe) { + throw new SynapseException("Invalid numeric value for interval.", nfe); + } catch (Exception e) { + throw new SynapseException("Invalid value for interval.", e); + } + this.sequential = true; + if (cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL) != null) { + this.sequential = Boolean + .parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL)); + } + this.coordination = true; + if (cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION) != null) { + this.coordination = Boolean.parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION)); + } + if (!inboundEpEventQueueMap.containsKey(this.name)) { + BlockingQueue> eventQueue = new LinkedBlockingQueue<>(); + inboundEpEventQueueMap.put(this.name, eventQueue); + } + } + + private void setProperties () { + LOGGER.info("Initializing the properties"); + try { + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE, FILE_OFFSET_STORAGE_CLASS); + } + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE).equals(FILE_OFFSET_STORAGE_CLASS)) { + String filePath; + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) == null) { + filePath = "cdc/offsetStorage/" + this.name + "_.dat"; + } else { + filePath = this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME); + } + createFile(filePath); + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) == null) { + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); + } + + String passwordString = this.cdcProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); + SynapseEnvironment synapseEnvironment = this.synapseEnvironment; + MessageContext messageContext = synapseEnvironment.createMessageContext(); + + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + + if (this.cdcProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) == null) { + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) == null) { + this.cdcProperties.setProperty(DEBEZIUM_TOPIC_PREFIX, this.name +"_topic"); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER) == null) { + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + } + if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER) == null) { + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + } + if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + } + if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) == null) { + this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, FILE_SCHEMA_HISTORY_STORAGE_CLASS); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL).equals(FILE_SCHEMA_HISTORY_STORAGE_CLASS)) { + String filePath; + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) == null) { + filePath = "cdc/schemaHistory/" + this.name + "_.dat"; + } else { + filePath = this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME); + } + createFile(filePath); + this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } catch (NullPointerException e) { + LOGGER.error("A required property value is not defined", e); + throw new RuntimeException(e); + } + } + + private void createFile (String filePath) throws IOException { + File file = new File(filePath); + file.getParentFile().mkdirs(); + if(!file.exists()) { + file.createNewFile(); + } + } + + private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { + if (passwordString == null) { + return null; + } + Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); + String resolvedValue = ""; + if (lookupMatcher.find()) { + Value expression; + String expressionStr = lookupMatcher.group(1); + try { + expression = new Value(new SynapseXPath(expressionStr)); + + } catch (JaxenException e) { + throw new SynapseException("Error while building the expression : " + expressionStr, e); + } + resolvedValue = expression.evaluateValue(messageContext); + if (StringUtils.isEmpty(resolvedValue)) { + LOGGER.warn("Found Empty value for expression : " + expression.getExpression()); + resolvedValue = ""; + } + } else { + resolvedValue = passwordString; + } + return resolvedValue; + } + + + /** + * This will be called at the time of synapse artifact deployment. + */ + public void init() { + LOGGER.info("Inbound CDC listener " + name + " starting ..."); + pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); + pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, + synapseEnvironment, cdcProperties)); + + DebeziumEngine> engine = DebeziumEngine.create(Json.class) + .using(this.cdcProperties) + .notifying(record -> { + try { + inboundEpEventQueueMap.get(this.name).offer(record, interval, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }).build(); + + executorService = Executors.newSingleThreadExecutor(); + executorService.execute(engine); + start(); + } + + + /** + * Register/start the schedule service + */ + public void start() { + InboundTask task = new CDCTask(pollingConsumer, interval); + start(task, ENDPOINT_POSTFIX); + } + + /** + * Remove inbound endpoints. + * + * @param removeTask Whether to remove scheduled task from the registry or not. + */ + @Override + public void destroy(boolean removeTask) { + if (removeTask) { + destroy(); + executorService.shutdown(); + } + } + + @Override + public void update() { + // This will not be called for inbound endpoints + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java deleted file mode 100644 index 7df87f60ff..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import io.debezium.engine.ChangeEvent; -import org.apache.axiom.om.OMAbstractFactory; -import org.apache.axiom.om.OMElement; -import org.apache.axiom.soap.SOAPEnvelope; -import org.apache.axiom.soap.SOAPFactory; -import org.apache.axiom.util.UIDGenerator; -import org.apache.axis2.AxisFault; -import org.apache.axis2.builder.Builder; -import org.apache.axis2.builder.BuilderUtil; -import org.apache.axis2.context.OperationContext; -import org.apache.axis2.context.ServiceContext; -import org.apache.axis2.description.InOutAxisOperation; -import org.apache.axis2.transport.TransportUtils; -import org.apache.commons.io.input.AutoCloseInputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; -import org.apache.synapse.SynapseConstants; -import org.apache.synapse.api.ApiConstants; -import org.apache.synapse.core.axis2.Axis2MessageContext; -import org.apache.synapse.core.axis2.MessageContextCreatorForAxis2; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.mediators.MediatorFaultHandler; -import org.apache.synapse.mediators.base.SequenceMediator; -import org.wso2.carbon.inbound.endpoint.osgi.service.ServiceReferenceHolder; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.IOException; - - -import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; -import static org.wso2.carbon.inbound.endpoint.common.Constants.TENANT_DOMAIN; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.*; - - -public class CDCSourceHandler { - - private int port; - private InboundProcessorParams params; - private static final String tenantDomain = SUPER_TENANT_DOMAIN_NAME; - private static final Log log = LogFactory.getLog(CDCSourceHandler.class); - private static final String contentType = "application/json"; - - public CDCSourceHandler(int port, InboundProcessorParams params) { - this.port = port; - this.params = params; - } - - public void requestReceived(ChangeEvent eventRecord) { - if (eventRecord == null || eventRecord.value() == null) { - log.debug("CDC Source Handler received empty event record"); - } else { - log.debug("CDC Source Handler request received"); - - MessageContext synCtx = null; - try { - - synCtx = getSynapseMessageContext(tenantDomain); - - CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); - synCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); - synCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); - synCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); - synCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); - - org.apache.axis2.context.MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) synCtx) - .getAxis2MessageContext(); - Builder builder = BuilderUtil.getBuilderFromSelector(contentType, axis2MsgCtx); - - if (builder != null) { - String serializedChangeEvent = cdcEventOutput.getOutputJsonPayload().toString(); - InputStream in = new AutoCloseInputStream( - new ByteArrayInputStream(serializedChangeEvent.getBytes())); - - OMElement documentElement = builder.processDocument(in, contentType, axis2MsgCtx); - synCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); - - if (log.isDebugEnabled()) { - log.debug("CDCEvent being injected to Sequence"); - } - injectForMediation(synCtx); - return; - } - - } catch (AxisFault e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - private static org.apache.axis2.context.MessageContext createAxis2MessageContext() { - org.apache.axis2.context.MessageContext axis2MsgCtx = new org.apache.axis2.context.MessageContext(); - axis2MsgCtx.setMessageID(UIDGenerator.generateURNString()); - axis2MsgCtx.setConfigurationContext( - ServiceReferenceHolder.getInstance().getConfigurationContextService().getServerConfigContext()); - axis2MsgCtx.setProperty(org.apache.axis2.context.MessageContext.CLIENT_API_NON_BLOCKING, Boolean.TRUE); - axis2MsgCtx.setServerSide(true); - - return axis2MsgCtx; - } - - private static org.apache.synapse.MessageContext createSynapseMessageContext(String tenantDomain) throws AxisFault { - org.apache.axis2.context.MessageContext axis2MsgCtx = createAxis2MessageContext(); - ServiceContext svcCtx = new ServiceContext(); - OperationContext opCtx = new OperationContext(new InOutAxisOperation(), svcCtx); - axis2MsgCtx.setServiceContext(svcCtx); - axis2MsgCtx.setOperationContext(opCtx); - - axis2MsgCtx.setProperty(TENANT_DOMAIN, tenantDomain); - - SOAPFactory fac = OMAbstractFactory.getSOAP11Factory(); - SOAPEnvelope envelope = fac.getDefaultEnvelope(); - axis2MsgCtx.setEnvelope(envelope); - return MessageContextCreatorForAxis2.getSynapseMessageContext(axis2MsgCtx); - } - - public org.apache.synapse.MessageContext getSynapseMessageContext(String tenantDomain) throws AxisFault { - MessageContext synCtx = createSynapseMessageContext(tenantDomain); - synCtx.setProperty(SynapseConstants.IS_INBOUND, true); - ((Axis2MessageContext) synCtx).getAxis2MessageContext().setProperty(SynapseConstants.IS_INBOUND, true); - - return synCtx; - } - - - private void injectForMediation(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = getFaultSequence(synCtx); - - MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); - synCtx.pushFaultHandler(mediatorFaultHandler); - if (log.isDebugEnabled()) { - log.debug("injecting message to sequence : " + params.getInjectingSeq()); - } - synCtx.setProperty("inbound.endpoint.name", params.getName()); - synCtx.setProperty(ApiConstants.API_CALLER, params.getName()); - - SequenceMediator injectingSequence = null; - if (params.getInjectingSeq() != null) { - injectingSequence = (SequenceMediator) synCtx.getSequence(params.getInjectingSeq()); - } - if (injectingSequence == null) { - injectingSequence = (SequenceMediator) synCtx.getMainSequence(); - } - - synCtx.getEnvironment().injectMessage(synCtx, injectingSequence); - - } - - private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = null; - if (params.getOnErrorSeq() != null) { - faultSequence = (SequenceMediator) synCtx.getSequence(params.getOnErrorSeq()); - } - if (faultSequence == null) { - faultSequence = (SequenceMediator) synCtx.getFaultSequence(); - } - return faultSequence; - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java new file mode 100644 index 0000000000..4795fab44a --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.core.SynapseEnvironment; +import org.wso2.carbon.inbound.endpoint.common.InboundTask; + +import java.util.Properties; + +/** + * CDCTask class is used to schedule tasks for inbound CDC processor when + * required (coordination==true) + */ +public class CDCTask extends InboundTask { + + private static final Log logger = LogFactory.getLog(CDCTask.class.getName()); + + private CDCPollingConsumer pollingConsumer; + + public CDCTask(CDCPollingConsumer pollingConsumer, long interval) { + logger.debug("CDC Task initialize."); + this.interval = interval; + this.pollingConsumer = pollingConsumer; + } + + protected void taskExecute() { + if (logger.isDebugEnabled()) { + logger.debug("CDC Task executing."); + } + pollingConsumer.execute(); + } + + @Override + public Properties getInboundProperties() { + return pollingConsumer.getInboundProperties(); + } + + public void init(SynapseEnvironment synapseEnvironment) { + if (logger.isDebugEnabled()) { + logger.debug("Initializing Task."); + } + } + + public void destroy() { + if (logger.isDebugEnabled()) { + logger.debug("Destroying Task. "); + } + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java index 156a414e74..c23931d9b2 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -1,27 +1,31 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.wso2.carbon.inbound.endpoint.protocol.cdc; -public class InboundCDCConstants { +class InboundCDCConstants { /** Inbound Endpoint Parameters **/ - public static final String CDC = "cdc"; - public static final String INBOUND_ENDPOINT_PARAMETER_CDC_PORT = "inbound.cdc.port"; - //debezium - public static final String DEBEZIUM_NAME = "name"; - public static final String DEBEZIUM_SNAPSHOT_MODE = "snapshot.mode"; - public static final String DEBEZIUM_MAX_THREADS = "snapshot.max.threads"; public static final String DEBEZIUM_OFFSET_STORAGE = "offset.storage"; public static final String DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME = "offset.storage.file.filename"; public static final String DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS = "offset.flush.interval.ms"; - public static final String DEBEZIUM_CONNECTOR_CLASS = "connector.class"; - public static final String DEBEZIUM_DATABASE_HOSTNAME = "database.hostname"; - public static final String DEBEZIUM_DATABASE_PORT = "database.port"; - public static final String DEBEZIUM_DATABASE_USER = "database.user"; public static final String DEBEZIUM_DATABASE_PASSWORD = "database.password"; - public static final String DEBEZIUM_DATABASE_DBNAME = "database.dbname"; - public static final String DEBEZIUM_TABLES_INCLUDE_LIST = "table.include.list"; - public static final String DEBEZIUM_OPERATIONS_EXCLUDE_LIST = "skipped.operations"; - public static final String DEBEZIUM_DATABASE_SERVER_ID = "database.server.id"; public static final String DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL = "database.allowPublicKeyRetrieval"; public static final String DEBEZIUM_TOPIC_PREFIX = "topic.prefix"; @@ -48,4 +52,5 @@ public class InboundCDCConstants { public static final String TABLE = "table"; public static final String TRUE = "true"; + } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java deleted file mode 100644 index 37e3fbd70d..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class InboundCDCEventExecutor { - - private ExecutorService executorService; - - public InboundCDCEventExecutor () { - executorService = Executors.newSingleThreadExecutor(); - } - - public ExecutorService getExecutorService() { - return this.executorService; - } - - public void shutdownEventExecutor() { - executorService.shutdown(); - try { - if (!executorService.awaitTermination(800, TimeUnit.MILLISECONDS)) { - executorService.shutdownNow(); - } - } catch (InterruptedException e) { - executorService.shutdownNow(); - } - } - - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties deleted file mode 100644 index 1da8230168..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Root logger option -log4j.rootLogger=INFO, stdout - -# Direct log messages to stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/pom.xml b/pom.xml index d184d4e4cf..0a87249f76 100644 --- a/pom.xml +++ b/pom.xml @@ -1104,6 +1104,12 @@ ${mysql.connector.version} test + + com.oracle.database.jdbc + ojdbc8 + ${ojdbc8.version} + test + org.apache.derby.wso2 derby @@ -1465,6 +1471,21 @@ debezium-connector-mysql ${debezium.version} + + io.debezium + debezium-connector-sqlserver + ${debezium.version} + + + io.debezium + debezium-connector-postgres + ${debezium.version} + + + io.debezium + debezium-connector-oracle + ${debezium.version} + io.debezium debezium-core @@ -1644,11 +1665,12 @@ 2.4.1.Final + 21.3.0.0 3.3.1 - 4.0.0-wso2v38 + 4.0.0-wso2v40 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 67883e384dcc041516d072cdbae5e3ded8d35b38 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Fri, 28 Jul 2023 13:28:17 +0530 Subject: [PATCH 25/47] Set the output format as json in a way a user cannot override Get the param as allowed.operations instead of skipped.operations Change the property names of the sequence Add isDebugEnable check where string concaterntions are done. Change Json library to Gson and resolve the project snapshot version Directly throw the exceptions without using fault sequence Add bundled dependencies --- .../pom.xml | 2 +- components/business-adaptors/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/crypto-service/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/data/data-services/pom.xml | 2 +- components/data/pom.xml | 2 +- components/javax.cache/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/data-publishers/pom.xml | 2 +- .../pom.xml | 2 +- .../utsecurity/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/extensions/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 29 ++-- .../endpoint/protocol/cdc/CDCEventOutput.java | 51 +++--- .../protocol/cdc/CDCInjectHandler.java | 150 +++++++----------- .../protocol/cdc/CDCPollingConsumer.java | 81 ++++++---- .../endpoint/protocol/cdc/CDCProcessor.java | 137 +++++++--------- .../endpoint/protocol/cdc/CDCTask.java | 12 +- .../protocol/cdc/InboundCDCConstants.java | 10 +- .../CDCInbound/cdc-inbound-endpoint.xml | 17 +- .../resources/CDCInbound/cdc_process_seq.xml | 8 +- .../mediation/inbound-endpoints/pom.xml | 2 +- .../org.wso2.carbon.mediator.cache/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/mediators/pom.xml | 2 +- components/mediation/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/registry/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/security/pom.xml | 2 +- .../startup/mediation-startup/pom.xml | 2 +- components/mediation/startup/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/tasks/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/transports/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 4 +- .../org.wso2.micro.integrator.core/pom.xml | 2 +- .../pom.xml | 2 +- .../org.wso2.micro.integrator.probes/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../org.wso2.micro.integrator.server/pom.xml | 2 +- .../org.wso2.micro.integrator.utils/pom.xml | 2 +- components/pom.xml | 2 +- distribution/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../data-services-feature/pom.xml | 2 +- features/data-features/pom.xml | 2 +- .../pom.xml | 2 +- .../builtin-mediators/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../data-publisher-features/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../inbound-endpoint/pom.xml | 2 +- .../mediation-features/ntask-feature/pom.xml | 2 +- features/mediation-features/pom.xml | 2 +- .../pom.xml | 2 +- .../security-features/pom.xml | 2 +- .../pom.xml | 2 +- .../transport-features/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- features/pom.xml | 2 +- features/wso2mi-hl7/pom.xml | 2 +- integration/automation-extensions/pom.xml | 2 +- integration/dataservice-hosting-tests/pom.xml | 2 +- .../samples/data-services/clients/pom.xml | 2 +- .../samples/data-services/pom.xml | 2 +- .../dataservice-hosting-tests/samples/pom.xml | 2 +- .../tests-common/admin-clients/pom.xml | 2 +- .../integration-test-utils/pom.xml | 2 +- .../tests-common/pom.xml | 2 +- .../tests-integration/pom.xml | 2 +- .../tests-integration/tests/pom.xml | 2 +- integration/management-api-tests/pom.xml | 2 +- .../mediation-tests/coverage-report/pom.xml | 2 +- integration/mediation-tests/pom.xml | 2 +- .../mediation-tests/service-samples/pom.xml | 2 +- .../mediation-tests/tests-http/pom.xml | 2 +- .../mediation-tests/tests-mediator-1/pom.xml | 2 +- .../mediation-tests/tests-mediator-2/pom.xml | 2 +- .../mediation-tests/tests-other/pom.xml | 2 +- .../tests-patches-with-smb2/pom.xml | 2 +- .../mediation-tests/tests-patches/pom.xml | 2 +- .../mediation-tests/tests-platform/pom.xml | 2 +- .../tests-platform/tests-coordination/pom.xml | 2 +- .../tests-platform/tests-rabbitmq/pom.xml | 2 +- .../tests-platform/tests-transaction/pom.xml | 2 +- .../tests-platform/tests-userstore/pom.xml | 2 +- .../mediation-tests/tests-sample/pom.xml | 2 +- .../mediation-tests/tests-service/pom.xml | 2 +- .../mediation-tests/tests-transport-2/pom.xml | 2 +- .../mediation-tests/tests-transport/pom.xml | 2 +- integration/pom.xml | 2 +- integration/samples/product/pom.xml | 2 +- .../tests-common/admin-clients/pom.xml | 2 +- .../integration-test-utils/pom.xml | 2 +- integration/tests-common/pom.xml | 2 +- p2-profile/pom.xml | 2 +- pom.xml | 130 ++------------- 129 files changed, 348 insertions(+), 517 deletions(-) diff --git a/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml b/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml index 0a69bbd758..8b59f85737 100644 --- a/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml +++ b/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml @@ -21,7 +21,7 @@ business-adaptors org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/business-adaptors/pom.xml b/components/business-adaptors/pom.xml index 6e0757b4f7..50855f5720 100644 --- a/components/business-adaptors/pom.xml +++ b/components/business-adaptors/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml b/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml index 56fd0070b6..dc4f5931e4 100644 --- a/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml +++ b/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml @@ -22,7 +22,7 @@ crypto-service org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml b/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml index 3be79fceb1..458b982aee 100644 --- a/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml +++ b/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml @@ -22,7 +22,7 @@ crypto-service org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/pom.xml b/components/crypto-service/pom.xml index 49edd9d603..2e1d0590e2 100644 --- a/components/crypto-service/pom.xml +++ b/components/crypto-service/pom.xml @@ -22,7 +22,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml index ca19abee70..44be98d96c 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml index 33b5cb3545..cee5e7c931 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml index a21c7c6ed9..90bf7482c0 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml index 3ff1f34f98..b21c1cd195 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml index 2b78aad02f..4f4db79bdd 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/pom.xml b/components/data/data-services/pom.xml index 3aecd10820..6f30cb90e3 100644 --- a/components/data/data-services/pom.xml +++ b/components/data/data-services/pom.xml @@ -21,7 +21,7 @@ data org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/pom.xml b/components/data/pom.xml index b66399951f..475ed65475 100644 --- a/components/data/pom.xml +++ b/components/data/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/javax.cache/pom.xml b/components/javax.cache/pom.xml index 746253fd9d..779c7dcb44 100644 --- a/components/javax.cache/pom.xml +++ b/components/javax.cache/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml index aff407c3d5..6fa41c7c08 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei data-publishers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml index c2b9773fdd..95504f6199 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei data-publishers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml index a953f5e679..db5bf3f675 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml @@ -21,7 +21,7 @@ data-publishers org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/pom.xml b/components/mediation/data-publishers/pom.xml index ea85db155c..73ea65aee4 100644 --- a/components/mediation/data-publishers/pom.xml +++ b/components/mediation/data-publishers/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml index 5f096b43c8..cdd74bbc72 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml @@ -21,7 +21,7 @@ extensions org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml index 6a5fd018b9..bc4c6f98b1 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei org.wso2.micro.integrator.security.handlers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml index 53da2fd2a4..a796cabfbc 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/extensions/pom.xml b/components/mediation/extensions/pom.xml index 392b190e51..569fe91a6d 100644 --- a/components/mediation/extensions/pom.xml +++ b/components/mediation/extensions/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml index ccfcb449a6..2231b1fd56 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml @@ -23,7 +23,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml index a3a81d1e2d..06a1fa1d5d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml @@ -21,7 +21,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml index bb7dd45d96..20b678602e 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml @@ -20,7 +20,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml @@ -170,6 +170,10 @@ org.apache.axis2.transport axis2-transport-rabbitmq-amqp + + org.wso2.orbit.debezium + debezium + org.apache.logging.log4j log4j-core @@ -181,24 +185,12 @@ test - org.wso2.carbon.mediation - org.wso2.carbon.inbound.endpoint.persistence - - - io.debezium - debezium-embedded - - - io.debezium - debezium-api - - - io.debezium - debezium-connector-mysql + com.google.code.gson + gson - org.slf4j - slf4j-api + org.wso2.carbon + org.wso2.carbon.securevault @@ -232,7 +224,8 @@ org.wso2.carbon.inbound.endpoint.persistence, - org.wso2.carbon.inbound.endpoint.osgi.service; + org.wso2.carbon.inbound.endpoint.osgi.service, + *;resolution:=optional * diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index ec3af80f6a..cdb52b0b9d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -18,8 +18,10 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import io.debezium.engine.ChangeEvent; -import org.json.JSONObject; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.AFTER; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.BEFORE; @@ -32,36 +34,27 @@ public class CDCEventOutput { - private JSONObject payload; + private JsonObject payload; private enum operations {c, r, u, d}; CDCEventOutput(ChangeEvent event) { String valueString = event.value().toString(); - JSONObject value = new JSONObject(valueString); - this.payload = value.getJSONObject(PAYLOAD); + JsonObject value = new Gson().fromJson(valueString, JsonObject.class); + this.payload = value.getAsJsonObject(PAYLOAD); } - public Object getJsonPayloadBeforeEvent() { - Object beforeObject = null; - if (payload.has(BEFORE)) { - beforeObject = payload.get(BEFORE); - } - return beforeObject; - + public JsonElement getJsonPayloadBeforeEvent() { + return payload.get(BEFORE); } - public Object getJsonPayloadAfterEvent() { - Object afterObject = null; - if (payload.has(AFTER)) { - afterObject = payload.get(AFTER); - } - return afterObject; + public JsonElement getJsonPayloadAfterEvent() { + return payload.get(AFTER); } public Long getTs_ms() { if (payload.has(TS_MS)) { - return payload.getLong(TS_MS); + return payload.get(TS_MS).getAsLong(); } return null; } @@ -69,15 +62,15 @@ public Long getTs_ms() { public String getDatabase() { if (getSource() != null) { if (getSource().has(DB)) { - return getSource().getString(DB); + return getSource().get(DB).getAsString(); } return null; } return null; } - public Object getTable() { - Object tableObject = null; + public JsonElement getTable() { + JsonElement tableObject = null; if (getSource() != null) { if (getSource().has(TABLE)) { tableObject = getSource().get(TABLE); @@ -86,16 +79,16 @@ public Object getTable() { return tableObject; } - private JSONObject getSource () { + private JsonObject getSource () { if (payload.has(SOURCE)) { - return payload.getJSONObject(SOURCE); + return payload.getAsJsonObject(SOURCE); } return null; } public String getOp() { if (payload.has(OP)) { - return getOpString(payload.getString(OP)); + return getOpString(payload.get(OP).getAsString()); } return null; } @@ -116,14 +109,14 @@ private String getOpString(String op) { return null; } - public JSONObject getOutputJsonPayload () { + public JsonObject getOutputJsonPayload () { if (payload == null) { return null; } - JSONObject jsonPayload = new JSONObject(); - jsonPayload.put(OP, getOp()); - jsonPayload.put(BEFORE, getJsonPayloadBeforeEvent()); - jsonPayload.put(AFTER, getJsonPayloadAfterEvent()); + JsonObject jsonPayload = new JsonObject(); + jsonPayload.addProperty(OP, getOp()); + jsonPayload.add(BEFORE, getJsonPayloadBeforeEvent()); + jsonPayload.add(AFTER, getJsonPayloadAfterEvent()); return jsonPayload; } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java index 00e5fe2127..6fa691502f 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java @@ -24,26 +24,22 @@ import org.apache.axis2.AxisFault; import org.apache.axis2.context.MessageContext; import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.synapse.SynapseConstants; -import org.apache.synapse.SynapseException; import org.apache.synapse.commons.json.JsonUtil; import org.apache.synapse.core.SynapseEnvironment; import org.apache.synapse.inbound.InboundEndpoint; -import org.apache.synapse.mediators.MediatorFaultHandler; import org.apache.synapse.mediators.base.SequenceMediator; import org.apache.synapse.transport.customlogsetter.CustomLogSetter; -import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericConstants; -import java.io.InputStream; -import java.util.Map; import java.util.Properties; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DATABASE_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OPERATIONS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLES; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_DATABASE_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_OPERATIONS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_TABLES; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_TS_MS; public class CDCInjectHandler { @@ -54,8 +50,6 @@ public class CDCInjectHandler { private boolean sequential; private Properties cdcProperties; private SynapseEnvironment synapseEnvironment; - private Map transportHeaders; - private static final String contentType = "application/json"; public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequential, SynapseEnvironment synapseEnvironment, Properties cdcProperties) { @@ -69,103 +63,76 @@ public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequenti /** * Inject the message to the sequence */ - public boolean invoke(Object object, String inboundEndpointName) throws SynapseException { + public boolean invoke(Object object, String inboundEndpointName) { ChangeEvent eventRecord = (ChangeEvent) object; if (eventRecord == null || eventRecord.value() == null) { + logger.debug("CDC Source Handler received empty event record"); + } else { + org.apache.synapse.MessageContext msgCtx = createMessageContext(); + msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); + msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); + msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); + InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); + CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + msgCtx.setProperty(CDC_DATABASE_NAME, cdcEventOutput.getDatabase()); + msgCtx.setProperty(CDC_TABLES, cdcEventOutput.getTable().toString()); + msgCtx.setProperty(CDC_OPERATIONS, cdcEventOutput.getOp()); + msgCtx.setProperty(CDC_TS_MS, cdcEventOutput.getTs_ms().toString()); + if (logger.isDebugEnabled()) { - logger.debug("CDC Source Handler received empty event record"); + logger.debug("Processed event : " + eventRecord); } - } else { - InputStream in = null; + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + + OMElement documentElement = null; try { - org.apache.synapse.MessageContext msgCtx = createMessageContext(); - msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); - msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); - msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); - InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); - CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); - - CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); - msgCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); - msgCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); - msgCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); - msgCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, + cdcEventOutput.getOutputJsonPayload().toString(), true, true); - if (logger.isDebugEnabled()) { - logger.debug("Processed event : " + eventRecord); - } - MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) - .getAxis2MessageContext(); - - //Builder builder = null; - OMElement documentElement = null; - try { - documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, - cdcEventOutput.getOutputJsonPayload().toString(), true, true); - - } catch (AxisFault ex) { - logger.error("Error while creating the OMElement", ex); - msgCtx.setProperty(SynapseConstants.ERROR_CODE, GenericConstants.INBOUND_BUILD_ERROR); - msgCtx.setProperty(SynapseConstants.ERROR_MESSAGE, ex.getMessage()); - SequenceMediator faultSequence = getFaultSequence(msgCtx); - faultSequence.mediate(msgCtx); - return true; - } + } catch (AxisFault ex) { + handleError("Error while creating the OMElement"); + } - // Inject the message to the sequence. + // Inject the message to the sequence. + try { msgCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); - if (injectingSeq == null || injectingSeq.equals("")) { - logger.error("Sequence name not specified. Sequence : " + injectingSeq); - return false; + } catch (AxisFault e) { + handleError("Error while creating the SOAP Envelop"); + } + + if (StringUtils.isBlank(injectingSeq)) { + handleError("Injecting sequence name not specified"); + } + SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() + .getSequence(injectingSeq); + if (seq != null) { + if (logger.isDebugEnabled()) { + logger.debug("injecting message to sequence : " + injectingSeq); } - SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() - .getSequence(injectingSeq); - if (seq != null) { - if (logger.isDebugEnabled()) { - logger.debug("injecting message to sequence : " + injectingSeq); - } - if (!seq.isInitialized()) { - seq.init(synapseEnvironment); - } - SequenceMediator faultSequence = getFaultSequence(msgCtx); - MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); - msgCtx.pushFaultHandler(mediatorFaultHandler); - - if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { - return false; - } - } else { - logger.error("Sequence: " + injectingSeq + " not found"); + if (!seq.isInitialized()) { + seq.init(synapseEnvironment); } - } catch (AxisFault e) { - throw new RuntimeException(e); + seq.setErrorHandler(onErrorSeq); + + if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { + handleError("Failed to inject the sequence"); + } + } else { + handleError("Sequence:" + injectingSeq + " not found"); } } return true; } - private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = null; - if (this.onErrorSeq != null) { - faultSequence = (SequenceMediator) synCtx.getSequence(this.onErrorSeq); - } - - if (faultSequence == null) { - faultSequence = (SequenceMediator) synCtx.getFaultSequence(); - } - - return faultSequence; - } - - - /** - * @param transportHeaders the transportHeaders to set - */ - public void setTransportHeaders(Map transportHeaders) { - this.transportHeaders = transportHeaders; - } + private void handleError(String msg) { + logger.error(msg); + throw new RuntimeException(msg); + } /** * Create the initial message context for the file @@ -177,7 +144,6 @@ private org.apache.synapse.MessageContext createMessageContext() { .getAxis2MessageContext(); axis2MsgCtx.setServerSide(true); axis2MsgCtx.setMessageID(UUIDGenerator.getUUID()); - axis2MsgCtx.setProperty(MessageContext.TRANSPORT_HEADERS, transportHeaders); msgCtx.setProperty(MessageContext.CLIENT_API_NON_BLOCKING, true); return msgCtx; } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java index dbd56d18f2..906982c1ac 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java @@ -18,15 +18,17 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.synapse.core.SynapseEnvironment; +import java.io.IOException; import java.util.Date; import java.util.Properties; -import java.util.concurrent.BlockingQueue; - -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor.inboundEpEventQueueMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * This class implement the processing logic related to inbound CDC protocol. @@ -42,6 +44,8 @@ public class CDCPollingConsumer { private long scanInterval; private Long lastRanTime; private CDCInjectHandler injectHandler; + private ExecutorService executorService = null; + private DebeziumEngine> engine = null; public CDCPollingConsumer(Properties cdcProperties, String inboundEndpointName, SynapseEnvironment synapseEnvironment, long scanInterval) { @@ -68,25 +72,21 @@ public void registerHandler(CDCInjectHandler injectHandler) { * interval. Timestamp based check is done to avoid that. */ public void execute() { - try { - if (logger.isDebugEnabled()) { - logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); - } - // Check if the cycles are running in correct interval and start - // scan - long currentTime = (new Date()).getTime(); - if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { - lastRanTime = currentTime; - poll(); - } else if (logger.isDebugEnabled()) { - logger.debug( - "Skip cycle since cuncurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); - } - if (logger.isDebugEnabled()) { - logger.debug("End : CDC Inbound EP : " + inboundEndpointName); - } - } catch (Exception e) { - logger.error("Error while getting events. " + e.getMessage(), e); + if (logger.isDebugEnabled()) { + logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); + } + // Check if the cycles are running in correct interval and start + // scan + long currentTime = (new Date()).getTime(); + if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { + lastRanTime = currentTime; + poll(); + } else if (logger.isDebugEnabled()) { + logger.debug( + "Skip cycle since concurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); + } + if (logger.isDebugEnabled()) { + logger.debug("End : CDC Inbound EP : " + inboundEndpointName); } } @@ -95,24 +95,41 @@ public void execute() { * according to the registered handler */ public ChangeEvent poll() { + logger.debug("Start : listening to DB events : "); + listenDataChanges(); + logger.debug("End : Listening to DB events : "); + return null; + } - if (logger.isDebugEnabled()) { - logger.debug("Start : listening to DB events : "); - } + private void listenDataChanges () { + executorService = Executors.newSingleThreadExecutor(); - BlockingQueue> eventQueue = inboundEpEventQueueMap.get(inboundEndpointName); - while (!eventQueue.isEmpty()) { - injectHandler.invoke(eventQueue.poll(), inboundEndpointName); - } + if (engine == null || executorService.isShutdown()) { + engine = DebeziumEngine.create(Json.class) + .using(this.cdcProperties) + .notifying(record -> { + injectHandler.invoke(record, this.inboundEndpointName); + }).build(); - if (logger.isDebugEnabled()) { - logger.debug("End : Listening to DB events : "); + executorService.execute(engine); } - return null; } protected Properties getInboundProperties() { return cdcProperties; } + protected void destroy () { + if (!executorService.isShutdown()) { + executorService.shutdown(); + } + try { + if (engine != null) { + engine.close(); + } + } catch (IOException e) { + throw new RuntimeException("Error while closing the Debezium Engine", e); + } + } + } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java index f5353e2acd..2219fbd80f 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -18,34 +18,27 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.format.Json; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; import org.apache.synapse.SynapseException; import org.apache.synapse.core.SynapseEnvironment; import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundTaskProcessor; -import org.apache.synapse.mediators.Value; import org.apache.synapse.task.TaskStartupObserver; -import org.apache.synapse.util.xpath.SynapseXPath; -import org.jaxen.JaxenException; +import org.apache.synapse.util.resolver.SecureVaultResolver; import org.wso2.carbon.inbound.endpoint.common.InboundRequestProcessorImpl; import org.wso2.carbon.inbound.endpoint.common.InboundTask; import org.wso2.carbon.inbound.endpoint.protocol.PollingConstants; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import java.util.Properties; -import java.util.concurrent.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_ALLOWED_OPERATIONS; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; @@ -58,6 +51,7 @@ import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SKIPPED_OPERATIONS; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; public class CDCProcessor extends InboundRequestProcessorImpl implements TaskStartupObserver, InboundTaskProcessor { @@ -67,15 +61,13 @@ public class CDCProcessor extends InboundRequestProcessorImpl implements TaskSta private String injectingSeq; private String onErrorSeq; private boolean sequential; - - private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; - private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); - protected static Map inboundEpEventQueueMap = new HashMap(); - private ExecutorService executorService = null; + + private enum operations {create, update, delete, truncate}; + private enum opCodes {c, u, d, t}; public CDCProcessor(InboundProcessorParams params) { this.name = params.getName(); @@ -88,8 +80,6 @@ public CDCProcessor(InboundProcessorParams params) { this.interval = Long.parseLong(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_INTERVAL)); } catch (NumberFormatException nfe) { throw new SynapseException("Invalid numeric value for interval.", nfe); - } catch (Exception e) { - throw new SynapseException("Invalid value for interval.", e); } this.sequential = true; if (cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL) != null) { @@ -100,10 +90,6 @@ public CDCProcessor(InboundProcessorParams params) { if (cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION) != null) { this.coordination = Boolean.parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION)); } - if (!inboundEpEventQueueMap.containsKey(this.name)) { - BlockingQueue> eventQueue = new LinkedBlockingQueue<>(); - inboundEpEventQueueMap.put(this.name, eventQueue); - } } private void setProperties () { @@ -129,30 +115,27 @@ private void setProperties () { String passwordString = this.cdcProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); SynapseEnvironment synapseEnvironment = this.synapseEnvironment; - MessageContext messageContext = synapseEnvironment.createMessageContext(); - this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, SecureVaultResolver.resolve(synapseEnvironment, passwordString)); if (this.cdcProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) == null) { this.cdcProperties.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); } + if (this.cdcProperties.getProperty(DEBEZIUM_ALLOWED_OPERATIONS) != null) { + this.cdcProperties.setProperty(DEBEZIUM_SKIPPED_OPERATIONS, + getSkippedOperationsString(this.cdcProperties.getProperty(DEBEZIUM_ALLOWED_OPERATIONS))); + } + if (this.cdcProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) == null) { this.cdcProperties.setProperty(DEBEZIUM_TOPIC_PREFIX, this.name +"_topic"); } - if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER) == null) { - this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - } - if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER) == null) { - this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - } - if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE) == null) { - this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); - } - if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE) == null) { - this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); - } + // set the output format as json in a way a user cannot override + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) == null) { this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, FILE_SCHEMA_HISTORY_STORAGE_CLASS); @@ -170,10 +153,9 @@ private void setProperties () { } } catch (IOException e) { - throw new RuntimeException(e); - } catch (NullPointerException e) { - LOGGER.error("A required property value is not defined", e); - throw new RuntimeException(e); + String msg = "Error While creating the SCHEMAHISTORY or OFFSET storage file"; + LOGGER.error(msg); + throw new RuntimeException(msg, e); } } @@ -185,32 +167,6 @@ private void createFile (String filePath) throws IOException { } } - private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { - if (passwordString == null) { - return null; - } - Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); - String resolvedValue = ""; - if (lookupMatcher.find()) { - Value expression; - String expressionStr = lookupMatcher.group(1); - try { - expression = new Value(new SynapseXPath(expressionStr)); - - } catch (JaxenException e) { - throw new SynapseException("Error while building the expression : " + expressionStr, e); - } - resolvedValue = expression.evaluateValue(messageContext); - if (StringUtils.isEmpty(resolvedValue)) { - LOGGER.warn("Found Empty value for expression : " + expression.getExpression()); - resolvedValue = ""; - } - } else { - resolvedValue = passwordString; - } - return resolvedValue; - } - /** * This will be called at the time of synapse artifact deployment. @@ -220,19 +176,6 @@ public void init() { pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, synapseEnvironment, cdcProperties)); - - DebeziumEngine> engine = DebeziumEngine.create(Json.class) - .using(this.cdcProperties) - .notifying(record -> { - try { - inboundEpEventQueueMap.get(this.name).offer(record, interval, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }).build(); - - executorService = Executors.newSingleThreadExecutor(); - executorService.execute(engine); start(); } @@ -254,7 +197,7 @@ public void start() { public void destroy(boolean removeTask) { if (removeTask) { destroy(); - executorService.shutdown(); + pollingConsumer.destroy(); } } @@ -262,4 +205,34 @@ public void destroy(boolean removeTask) { public void update() { // This will not be called for inbound endpoints } + + private String getOpCode(String op) { + if (op != null) { + switch (operations.valueOf(op)) { + case create: + return opCodes.c.toString(); + case update: + return opCodes.u.toString(); + case delete: + return opCodes.d.toString(); + case truncate: + return opCodes.t.toString(); + } + } + return ""; + } + + /** + * Get the comma separated list containing allowed operations and returns the string of skipped operation codes + * @param allowedOperationsString string + * @return the coma separated string of skipped operation codes + */ + private String getSkippedOperationsString(String allowedOperationsString) { + List allOperations = Stream.of(opCodes.values()).map(Enum :: toString).collect(Collectors.toList()); + Set allowedOperationsSet = Stream.of(allowedOperationsString.split(",")). + map(String :: trim).map(String :: toLowerCase).map(op -> getOpCode(op)). + collect(Collectors.toSet()); + allOperations.removeAll(allowedOperationsSet); + return String.join(",", allOperations); + } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java index 4795fab44a..a81d0b6fa9 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java @@ -42,9 +42,7 @@ public CDCTask(CDCPollingConsumer pollingConsumer, long interval) { } protected void taskExecute() { - if (logger.isDebugEnabled()) { - logger.debug("CDC Task executing."); - } + logger.debug("CDC Task executing."); pollingConsumer.execute(); } @@ -54,14 +52,10 @@ public Properties getInboundProperties() { } public void init(SynapseEnvironment synapseEnvironment) { - if (logger.isDebugEnabled()) { - logger.debug("Initializing Task."); - } + logger.debug("Initializing Task."); } public void destroy() { - if (logger.isDebugEnabled()) { - logger.debug("Destroying Task. "); - } + logger.debug("Destroying Task. "); } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java index c23931d9b2..386cb7bae7 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -36,11 +36,15 @@ class InboundCDCConstants { public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL = "schema.history.internal"; public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME = "schema.history.internal.file.filename"; + public static final String DEBEZIUM_SKIPPED_OPERATIONS = "skipped.operations"; + public static final String DEBEZIUM_ALLOWED_OPERATIONS = "allowed.operations"; + /** Output Properties **/ - public static final String DATABASE_NAME = "database"; - public static final String TABLES ="tables"; - public static final String OPERATIONS ="operations"; + public static final String CDC_DATABASE_NAME = "cdc.database"; + public static final String CDC_TABLES ="cdc.tables"; + public static final String CDC_OPERATIONS ="cdc.operations"; + public static final String CDC_TS_MS = "cdc.ts_ms"; public static final String TS_MS = "ts_ms"; public static final String BEFORE = "before"; diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml index e3caf7bc88..606e5c8876 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml @@ -1,19 +1,22 @@ - 9091 + 1000 engine + initial org.apache.kafka.connect.storage.FileOffsetBackingStore - + cdc/offsetStorage/offsets1_.dat io.debezium.connector.mysql.MySqlConnector localhost 3306 root - rusiri@wso2 + {wso2:vault-lookup('mysql_password')} students - 85744 + 8574444 + server_1 + topic2 io.debezium.storage.file.history.FileSchemaHistory - + cdc/schemaHistory/schema_history1_.dat students.marks - c + create - \ No newline at end of file + diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml index 16bab87ac4..982d9998d9 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml @@ -2,15 +2,15 @@ - + - + - + - + diff --git a/components/mediation/inbound-endpoints/pom.xml b/components/mediation/inbound-endpoints/pom.xml index 66833fbb3c..5ce4036683 100644 --- a/components/mediation/inbound-endpoints/pom.xml +++ b/components/mediation/inbound-endpoints/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml b/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml index 4e16af2941..cdc613863b 100644 --- a/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml +++ b/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml @@ -19,7 +19,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml b/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml index dff14d9ad2..efb477fd35 100644 --- a/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml +++ b/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml @@ -19,7 +19,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml b/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml index 5394cf59e8..7a35d3c5e7 100644 --- a/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml +++ b/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediators - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml b/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml index 81d0947c06..2009bf2ad2 100644 --- a/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml +++ b/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml @@ -18,7 +18,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/pom.xml b/components/mediation/mediators/pom.xml index 59ac91f643..4c2eca448b 100644 --- a/components/mediation/mediators/pom.xml +++ b/components/mediation/mediators/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/pom.xml b/components/mediation/pom.xml index 7106440b85..e1b09e5e05 100644 --- a/components/mediation/pom.xml +++ b/components/mediation/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml b/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml index 518dda038c..20666dc9ee 100644 --- a/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml +++ b/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml @@ -21,7 +21,7 @@ registry org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/registry/pom.xml b/components/mediation/registry/pom.xml index 32ad58347c..c4d8540be5 100644 --- a/components/mediation/registry/pom.xml +++ b/components/mediation/registry/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml b/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml index b1b8b2d4df..975d584d03 100644 --- a/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml +++ b/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei security - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/components/mediation/security/pom.xml b/components/mediation/security/pom.xml index 1aec46d0fe..83fed6869e 100644 --- a/components/mediation/security/pom.xml +++ b/components/mediation/security/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/components/mediation/startup/mediation-startup/pom.xml b/components/mediation/startup/mediation-startup/pom.xml index e28101052e..f1c45d8b02 100644 --- a/components/mediation/startup/mediation-startup/pom.xml +++ b/components/mediation/startup/mediation-startup/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei startup - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/startup/pom.xml b/components/mediation/startup/pom.xml index cc09b17595..c8395abe36 100644 --- a/components/mediation/startup/pom.xml +++ b/components/mediation/startup/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml b/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml index a83fbc44fe..1444c7a9f4 100644 --- a/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml +++ b/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml @@ -21,7 +21,7 @@ tasks org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml b/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml index 5f076c79d6..caa1ea1f69 100644 --- a/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml +++ b/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml @@ -21,7 +21,7 @@ tasks org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/pom.xml b/components/mediation/tasks/pom.xml index 0453ce0ecd..103f572b9f 100644 --- a/components/mediation/tasks/pom.xml +++ b/components/mediation/tasks/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml b/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml index 3c07ca7f05..5c49872413 100644 --- a/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml +++ b/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei transports - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/transports/pom.xml b/components/mediation/transports/pom.xml index ff4e8043fd..b146732d53 100644 --- a/components/mediation/transports/pom.xml +++ b/components/mediation/transports/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.bootstrap/pom.xml b/components/org.wso2.micro.integrator.bootstrap/pom.xml index 832ce0bfbe..1a44dd26f9 100644 --- a/components/org.wso2.micro.integrator.bootstrap/pom.xml +++ b/components/org.wso2.micro.integrator.bootstrap/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.coordination/pom.xml b/components/org.wso2.micro.integrator.coordination/pom.xml index f20eaa0e85..50a65e072c 100644 --- a/components/org.wso2.micro.integrator.coordination/pom.xml +++ b/components/org.wso2.micro.integrator.coordination/pom.xml @@ -20,14 +20,14 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 org.wso2.micro.integrator.coordination bundle - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT org.wso2.ei diff --git a/components/org.wso2.micro.integrator.core/pom.xml b/components/org.wso2.micro.integrator.core/pom.xml index 037e10722d..28b06c6eb5 100644 --- a/components/org.wso2.micro.integrator.core/pom.xml +++ b/components/org.wso2.micro.integrator.core/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml index 5173d84d66..796ec2f9dc 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei org.wso2.micro.integrator.extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml index 286307adf9..4f6b438c97 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei org.wso2.micro.integrator.extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/pom.xml b/components/org.wso2.micro.integrator.extensions/pom.xml index fb0db4285d..5423075175 100644 --- a/components/org.wso2.micro.integrator.extensions/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.initializer/pom.xml b/components/org.wso2.micro.integrator.initializer/pom.xml index 56150bcc78..55c56f25b6 100755 --- a/components/org.wso2.micro.integrator.initializer/pom.xml +++ b/components/org.wso2.micro.integrator.initializer/pom.xml @@ -24,7 +24,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml b/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml index 10a697aab5..517b5a895d 100644 --- a/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml +++ b/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.logging.updater/pom.xml b/components/org.wso2.micro.integrator.logging.updater/pom.xml index 8c58da3ea4..bcb41a3603 100644 --- a/components/org.wso2.micro.integrator.logging.updater/pom.xml +++ b/components/org.wso2.micro.integrator.logging.updater/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml b/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml index 5439d19261..9ae38e9def 100644 --- a/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.common/pom.xml b/components/org.wso2.micro.integrator.ndatasource.common/pom.xml index 5dc73c3e64..9a1c25c982 100644 --- a/components/org.wso2.micro.integrator.ndatasource.common/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.common/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.core/pom.xml b/components/org.wso2.micro.integrator.ndatasource.core/pom.xml index 4c96487cb4..5d0e141e5f 100644 --- a/components/org.wso2.micro.integrator.ndatasource.core/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.core/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml b/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml index 35a65f4a7e..5b369f3f0f 100644 --- a/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.security/pom.xml b/components/org.wso2.micro.integrator.security/pom.xml index f0176c0e5d..a196f2a3d8 100644 --- a/components/org.wso2.micro.integrator.security/pom.xml +++ b/components/org.wso2.micro.integrator.security/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.server/pom.xml b/components/org.wso2.micro.integrator.server/pom.xml index 7df41940ec..df77746ccd 100644 --- a/components/org.wso2.micro.integrator.server/pom.xml +++ b/components/org.wso2.micro.integrator.server/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.utils/pom.xml b/components/org.wso2.micro.integrator.utils/pom.xml index 7f0de07457..c245e25e1c 100644 --- a/components/org.wso2.micro.integrator.utils/pom.xml +++ b/components/org.wso2.micro.integrator.utils/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/pom.xml b/components/pom.xml index 4e79757887..8f9f7eaabb 100644 --- a/components/pom.xml +++ b/components/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/distribution/pom.xml b/distribution/pom.xml index c98a375b15..411a80bbf0 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml b/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml index ee8d87b051..7817c6489c 100644 --- a/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml +++ b/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml b/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml index 80063f6e62..906e496f7e 100644 --- a/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml +++ b/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei data-services-feature - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/data-features/data-services-feature/pom.xml b/features/data-features/data-services-feature/pom.xml index a6e04fa737..d73f25cd47 100644 --- a/features/data-features/data-services-feature/pom.xml +++ b/features/data-features/data-services-feature/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei data-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/data-features/pom.xml b/features/data-features/pom.xml index e99acc9081..78d0fac8fe 100644 --- a/features/data-features/pom.xml +++ b/features/data-features/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml b/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml index 9d8673499b..179f499a89 100644 --- a/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml +++ b/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei builtin-mediators - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/builtin-mediators/pom.xml b/features/mediation-features/builtin-mediators/pom.xml index b988d2485a..8083b0e4d9 100644 --- a/features/mediation-features/builtin-mediators/pom.xml +++ b/features/mediation-features/builtin-mediators/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml index 2604162d08..411119de59 100644 --- a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml +++ b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml @@ -22,7 +22,7 @@ data-publisher-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml index d86b3960cb..bb7e55a1c7 100644 --- a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml +++ b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml @@ -21,7 +21,7 @@ data-publisher-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/pom.xml b/features/mediation-features/data-publisher-features/pom.xml index 7258167c63..79ef4f8d1e 100644 --- a/features/mediation-features/data-publisher-features/pom.xml +++ b/features/mediation-features/data-publisher-features/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml b/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml index 0c788efecb..c59c4179bf 100644 --- a/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml +++ b/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml @@ -20,7 +20,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml b/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml index 2a1af54e87..5c865654a0 100644 --- a/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml +++ b/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml @@ -20,7 +20,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml b/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml index 3cd44e86e8..8e22c3533a 100644 --- a/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml +++ b/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml @@ -22,7 +22,7 @@ inbound-endpoint org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/inbound-endpoint/pom.xml b/features/mediation-features/inbound-endpoint/pom.xml index 19625648bf..50975f771a 100644 --- a/features/mediation-features/inbound-endpoint/pom.xml +++ b/features/mediation-features/inbound-endpoint/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/ntask-feature/pom.xml b/features/mediation-features/ntask-feature/pom.xml index 13f2fd644f..959c45d7aa 100644 --- a/features/mediation-features/ntask-feature/pom.xml +++ b/features/mediation-features/ntask-feature/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/pom.xml b/features/mediation-features/pom.xml index e09b63d26c..6d38fe1b47 100644 --- a/features/mediation-features/pom.xml +++ b/features/mediation-features/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml b/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml index 0db9cea038..173d908fb0 100644 --- a/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml +++ b/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei security-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/security-features/pom.xml b/features/mediation-features/security-features/pom.xml index e6cb585676..add4a4717b 100644 --- a/features/mediation-features/security-features/pom.xml +++ b/features/mediation-features/security-features/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml b/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml index 2f2abf0081..4063dc455f 100644 --- a/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml +++ b/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei transport-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/transport-features/pom.xml b/features/mediation-features/transport-features/pom.xml index 65fd33e73b..93697a9c02 100644 --- a/features/mediation-features/transport-features/pom.xml +++ b/features/mediation-features/transport-features/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/features/org.wso2.micro.integrator.initializer.feature/pom.xml b/features/org.wso2.micro.integrator.initializer.feature/pom.xml index 81946ea1fc..4098e2f2d2 100644 --- a/features/org.wso2.micro.integrator.initializer.feature/pom.xml +++ b/features/org.wso2.micro.integrator.initializer.feature/pom.xml @@ -24,7 +24,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml b/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml index c8c7659742..adca9f84eb 100644 --- a/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml +++ b/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml b/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml index 81944ae1a1..b4ca053a8c 100644 --- a/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml +++ b/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.server.feature/pom.xml b/features/org.wso2.micro.integrator.server.feature/pom.xml index 46855f9ed7..502e0d720d 100644 --- a/features/org.wso2.micro.integrator.server.feature/pom.xml +++ b/features/org.wso2.micro.integrator.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/pom.xml b/features/pom.xml index ed4d6e342c..1b7701de8d 100644 --- a/features/pom.xml +++ b/features/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/wso2mi-hl7/pom.xml b/features/wso2mi-hl7/pom.xml index e4953ed90a..f81d709ec6 100644 --- a/features/wso2mi-hl7/pom.xml +++ b/features/wso2mi-hl7/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/integration/automation-extensions/pom.xml b/integration/automation-extensions/pom.xml index f4c18501ef..1d7fc44da2 100644 --- a/integration/automation-extensions/pom.xml +++ b/integration/automation-extensions/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/pom.xml b/integration/dataservice-hosting-tests/pom.xml index 53e71d7b21..b902e4cb8f 100755 --- a/integration/dataservice-hosting-tests/pom.xml +++ b/integration/dataservice-hosting-tests/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml b/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml index bc5cd14f59..8f1525b1ac 100755 --- a/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml +++ b/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservices-samples-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/data-services/pom.xml b/integration/dataservice-hosting-tests/samples/data-services/pom.xml index d8a0f2c12f..00b06d17b8 100755 --- a/integration/dataservice-hosting-tests/samples/data-services/pom.xml +++ b/integration/dataservice-hosting-tests/samples/data-services/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei mi-samples-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/pom.xml b/integration/dataservice-hosting-tests/samples/pom.xml index 0a7f4b11c3..12e4a59f98 100644 --- a/integration/dataservice-hosting-tests/samples/pom.xml +++ b/integration/dataservice-hosting-tests/samples/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml b/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml index fdcec13e4c..d3073f425c 100644 --- a/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml b/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml index 6d650f17dc..30217d7110 100644 --- a/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/pom.xml b/integration/dataservice-hosting-tests/tests-common/pom.xml index 76a9dcecfd..a202590487 100644 --- a/integration/dataservice-hosting-tests/tests-common/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-integration/pom.xml b/integration/dataservice-hosting-tests/tests-integration/pom.xml index b32dd14b3b..ae3adb7777 100644 --- a/integration/dataservice-hosting-tests/tests-integration/pom.xml +++ b/integration/dataservice-hosting-tests/tests-integration/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml b/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml index cf30ba11d1..fe23e4dfd6 100644 --- a/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml +++ b/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/management-api-tests/pom.xml b/integration/management-api-tests/pom.xml index 8899308fc5..4113715ef6 100644 --- a/integration/management-api-tests/pom.xml +++ b/integration/management-api-tests/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/coverage-report/pom.xml b/integration/mediation-tests/coverage-report/pom.xml index 7298ac2e8e..f77e6fd99e 100644 --- a/integration/mediation-tests/coverage-report/pom.xml +++ b/integration/mediation-tests/coverage-report/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/pom.xml b/integration/mediation-tests/pom.xml index 90c7489f19..917145041a 100755 --- a/integration/mediation-tests/pom.xml +++ b/integration/mediation-tests/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/service-samples/pom.xml b/integration/mediation-tests/service-samples/pom.xml index 82f8b58244..5548c7c112 100644 --- a/integration/mediation-tests/service-samples/pom.xml +++ b/integration/mediation-tests/service-samples/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-http/pom.xml b/integration/mediation-tests/tests-http/pom.xml index 3d28f17abe..1064a56354 100644 --- a/integration/mediation-tests/tests-http/pom.xml +++ b/integration/mediation-tests/tests-http/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-mediator-1/pom.xml b/integration/mediation-tests/tests-mediator-1/pom.xml index 011c9fd680..929a237385 100644 --- a/integration/mediation-tests/tests-mediator-1/pom.xml +++ b/integration/mediation-tests/tests-mediator-1/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-mediator-2/pom.xml b/integration/mediation-tests/tests-mediator-2/pom.xml index b3c728e6a1..630120bd26 100644 --- a/integration/mediation-tests/tests-mediator-2/pom.xml +++ b/integration/mediation-tests/tests-mediator-2/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-other/pom.xml b/integration/mediation-tests/tests-other/pom.xml index 98813da842..18296c04c3 100644 --- a/integration/mediation-tests/tests-other/pom.xml +++ b/integration/mediation-tests/tests-other/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-patches-with-smb2/pom.xml b/integration/mediation-tests/tests-patches-with-smb2/pom.xml index 6bf367e418..81573eeec3 100644 --- a/integration/mediation-tests/tests-patches-with-smb2/pom.xml +++ b/integration/mediation-tests/tests-patches-with-smb2/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-patches/pom.xml b/integration/mediation-tests/tests-patches/pom.xml index 784a735ac8..8cef21f449 100644 --- a/integration/mediation-tests/tests-patches/pom.xml +++ b/integration/mediation-tests/tests-patches/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/pom.xml b/integration/mediation-tests/tests-platform/pom.xml index 49ebf460de..e30198d2f3 100644 --- a/integration/mediation-tests/tests-platform/pom.xml +++ b/integration/mediation-tests/tests-platform/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-coordination/pom.xml b/integration/mediation-tests/tests-platform/tests-coordination/pom.xml index 27bf436a78..77272c9161 100644 --- a/integration/mediation-tests/tests-platform/tests-coordination/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-coordination/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml b/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml index ef2f503463..2809a29834 100644 --- a/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei org.wso2.carbon.ei.tests.platform - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-transaction/pom.xml b/integration/mediation-tests/tests-platform/tests-transaction/pom.xml index 405387b6ec..1beeadb05a 100644 --- a/integration/mediation-tests/tests-platform/tests-transaction/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-transaction/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-userstore/pom.xml b/integration/mediation-tests/tests-platform/tests-userstore/pom.xml index f05ab2f352..1794ebcaed 100644 --- a/integration/mediation-tests/tests-platform/tests-userstore/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-userstore/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-sample/pom.xml b/integration/mediation-tests/tests-sample/pom.xml index 52693e1d41..f7b95f465e 100644 --- a/integration/mediation-tests/tests-sample/pom.xml +++ b/integration/mediation-tests/tests-sample/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-service/pom.xml b/integration/mediation-tests/tests-service/pom.xml index bdd177c55e..acfb7cea56 100644 --- a/integration/mediation-tests/tests-service/pom.xml +++ b/integration/mediation-tests/tests-service/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-transport-2/pom.xml b/integration/mediation-tests/tests-transport-2/pom.xml index 3db8fdbda6..0bf1a4de2d 100644 --- a/integration/mediation-tests/tests-transport-2/pom.xml +++ b/integration/mediation-tests/tests-transport-2/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-transport/pom.xml b/integration/mediation-tests/tests-transport/pom.xml index 1edac05e38..bf35eba37c 100644 --- a/integration/mediation-tests/tests-transport/pom.xml +++ b/integration/mediation-tests/tests-transport/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/pom.xml b/integration/pom.xml index ba17ee36b6..69de98222f 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/samples/product/pom.xml b/integration/samples/product/pom.xml index b410214473..5b0c180582 100755 --- a/integration/samples/product/pom.xml +++ b/integration/samples/product/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/tests-common/admin-clients/pom.xml b/integration/tests-common/admin-clients/pom.xml index 4ecd4529ae..859491eb5c 100644 --- a/integration/tests-common/admin-clients/pom.xml +++ b/integration/tests-common/admin-clients/pom.xml @@ -5,7 +5,7 @@ org.wso2.ei integration-test-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/tests-common/integration-test-utils/pom.xml b/integration/tests-common/integration-test-utils/pom.xml index b3397cce83..d89a703ca8 100644 --- a/integration/tests-common/integration-test-utils/pom.xml +++ b/integration/tests-common/integration-test-utils/pom.xml @@ -8,7 +8,7 @@ org.wso2.ei integration-test-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/tests-common/pom.xml b/integration/tests-common/pom.xml index c786674e20..9d767f963e 100644 --- a/integration/tests-common/pom.xml +++ b/integration/tests-common/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/p2-profile/pom.xml b/p2-profile/pom.xml index 9a7f9e383c..89edb8aaec 100644 --- a/p2-profile/pom.xml +++ b/p2-profile/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 0a87249f76..83c0ef3a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -1104,12 +1104,6 @@ ${mysql.connector.version} test - - com.oracle.database.jdbc - ojdbc8 - ${ojdbc8.version} - test - org.apache.derby.wso2 derby @@ -1268,7 +1262,7 @@ org.wso2.ei org.wso2.micro.integrator.coordination - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT @@ -1457,116 +1451,14 @@ ${opencensus.orbit.version} - io.debezium - debezium-embedded - ${debezium.version} - - - io.debezium - debezium-api - ${debezium.version} - - - io.debezium - debezium-connector-mysql - ${debezium.version} - - - io.debezium - debezium-connector-sqlserver - ${debezium.version} - - - io.debezium - debezium-connector-postgres - ${debezium.version} - - - io.debezium - debezium-connector-oracle - ${debezium.version} - - - io.debezium - debezium-core - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-storage-kafka - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-storage-file - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-ddl-parser + org.wso2.orbit.debezium + debezium ${debezium.version} - - - org.slf4j - slf4j-api - - - - - com.github.shyiko - mysql-binlog-connector-java - 0.27.2 - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - org.slf4j - slf4j-api - - - - - org.apache.kafka - connect-runtime - ${kafka.version} - - - org.slf4j - slf4j-api - - - org.apache.kafka - connect-json - ${kafka.version} - - - org.slf4j - slf4j-api - - + com.google.code.gson + gson + ${version.com.google.code.gson} @@ -1663,14 +1555,10 @@ 2.3.0 2.3.1 - - 2.4.1.Final - 21.3.0.0 - - - 3.3.1 + + 2.1.4.Final.wso2v1 - 4.0.0-wso2v40 + 4.0.0-wso2v47 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 2df82c43ecc1390fcd0e337d0cddc5830f870446 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Wed, 20 Sep 2023 16:47:19 +0530 Subject: [PATCH 26/47] Resolve review comments --- .../inbound/endpoint/protocol/cdc/CDCEventOutput.java | 11 +++-------- .../inbound/endpoint/protocol/cdc/CDCProcessor.java | 10 +++++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index cdb52b0b9d..71e143eeb4 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -60,21 +60,16 @@ public Long getTs_ms() { } public String getDatabase() { - if (getSource() != null) { - if (getSource().has(DB)) { + if (getSource() != null && getSource().has(DB)) { return getSource().get(DB).getAsString(); - } - return null; } return null; } public JsonElement getTable() { JsonElement tableObject = null; - if (getSource() != null) { - if (getSource().has(TABLE)) { - tableObject = getSource().get(TABLE); - } + if (getSource() != null && getSource().has(TABLE)) { + tableObject = getSource().get(TABLE); } return tableObject; } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java index 2219fbd80f..3563d2f01b 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -64,7 +64,7 @@ public class CDCProcessor extends InboundRequestProcessorImpl implements TaskSta private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; - private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); + private static final Log logger = LogFactory.getLog(CDCProcessor.class); private enum operations {create, update, delete, truncate}; private enum opCodes {c, u, d, t}; @@ -93,7 +93,7 @@ public CDCProcessor(InboundProcessorParams params) { } private void setProperties () { - LOGGER.info("Initializing the properties"); + logger.info("Initializing the CDC properties"); try { if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) == null) { this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE, FILE_OFFSET_STORAGE_CLASS); @@ -153,8 +153,8 @@ private void setProperties () { } } catch (IOException e) { - String msg = "Error While creating the SCHEMAHISTORY or OFFSET storage file"; - LOGGER.error(msg); + String msg = "Error while setting the CDC Properties"; + logger.error(msg); throw new RuntimeException(msg, e); } } @@ -172,7 +172,7 @@ private void createFile (String filePath) throws IOException { * This will be called at the time of synapse artifact deployment. */ public void init() { - LOGGER.info("Inbound CDC listener " + name + " starting ..."); + logger.info("Inbound CDC listener " + name + " starting ..."); pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, synapseEnvironment, cdcProperties)); From 19d2afa2c95eb36de1985e98b450a4377f7fc680 Mon Sep 17 00:00:00 2001 From: malakaganga Date: Thu, 21 Sep 2023 11:12:08 +0530 Subject: [PATCH 27/47] Fix Service Catalog not publishing SOAP Proxy The specific problem arises when the WSDLReader component in APIM attempts to read the WSDL content with a schemaLocation attribute set to 'proxy2?xsd=sample.xsd'. Upon parsing, WSDLReader treats this schemaLocation as a file path rather than as a query string, leading to a failure in resolving the actual schema location. So changed the schemaLocation into actual URL. Fixes: https://github.com/wso2/micro-integrator/issues/2967 --- .../utils/ServiceCatalogUtils.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java index 1a49bb3731..bc81f3fea0 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java @@ -30,6 +30,7 @@ import org.apache.synapse.commons.resolvers.ResolverException; import org.apache.synapse.commons.resolvers.SystemResolver; import org.apache.synapse.config.SynapseConfigUtils; +import org.apache.synapse.core.axis2.ProxyService; import org.wso2.carbon.securevault.SecretCallbackHandlerService; import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.application.deployer.AppDeployerUtils; @@ -818,6 +819,9 @@ private static String getFileChecksum(File file) throws NoSuchAlgorithmException * @return WSDL file creation result. */ private static boolean readProxyServiceWSDL(File metadataYamlFile, File storeLocation) { + Collection proxyTable = + SynapseConfigUtils.getSynapseConfiguration( + org.wso2.micro.core.Constants.SUPER_TENANT_DOMAIN_NAME).getProxyServices(); BufferedReader bufferedReader = null; try { String proxyServiceUrl = getProxyServiceUrlFromMetadata(metadataYamlFile); @@ -843,8 +847,17 @@ private static boolean readProxyServiceWSDL(File metadataYamlFile, File storeLoc return false; } if (storeLocation.exists()) { + String wsdlString = responseWSDL.toString(); + boolean shouldSchemaLocationBeChanged = shouldSchemaLocationBeChanged(storeLocation, proxyTable); + if (shouldSchemaLocationBeChanged) { + // Replace schemaLocation values ending with .xsd and change everything up to ? to xyz using regex + String regexPattern = "(schemaLocation=\")[^\"?]*\\?(.*\\.xsd\")"; + String baseUrl = proxyServiceUrl.replace(WSDL_URL_PATH, "?"); + String replacement = "$1" + baseUrl + "$2"; + wsdlString = wsdlString.replaceAll(regexPattern, replacement); + } Files.write(Paths.get(storeLocation.getAbsolutePath(), WSDL_FILE_NAME), - responseWSDL.toString().getBytes()); + wsdlString.getBytes()); return true; } } catch (IOException e) { @@ -862,6 +875,23 @@ private static boolean readProxyServiceWSDL(File metadataYamlFile, File storeLoc return false; } + private static boolean shouldSchemaLocationBeChanged(File storeLocation, Collection proxyTable) { + String metaFileName = storeLocation.getName(); + String proxyServiceName = metaFileName.substring(0, + metaFileName.indexOf(PROXY_SERVICE_SUFFIX + METADATA_FOLDER_STRING)); + if (proxyTable != null && !proxyTable.isEmpty()) { + for (Object proxy : proxyTable) { + if (proxy instanceof ProxyService) { + ProxyService proxyService = (ProxyService) proxy; + if (proxyService.getName().equals(proxyServiceName) && proxyService.getResourceMap() != null) { + return true; + } + } + } + } + return false; + } + /** * Reads service URL from metadata file. * From 6f63f5f778ab8925d28f23b357c1a39081b672c0 Mon Sep 17 00:00:00 2001 From: Sajinie Date: Mon, 9 Oct 2023 10:46:36 +0530 Subject: [PATCH 28/47] Add a missing system property in micro-integrator.bat file Fixes https://github.com/wso2/micro-integrator/issues/2982 --- distribution/src/scripts/micro-integrator.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/scripts/micro-integrator.bat b/distribution/src/scripts/micro-integrator.bat index 7b048ea28a..4cfb26fe1e 100644 --- a/distribution/src/scripts/micro-integrator.bat +++ b/distribution/src/scripts/micro-integrator.bat @@ -197,7 +197,7 @@ if "%profileSet%" == "false" ( set profile=-Dprofile=micro-integrator-default ) -set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof" -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% -Djava.endorsed.dirs=%JAVA_ENDORSED% -DandesConfig=broker.xml -Dcarbon.registry.root=/ -Dcarbon.home="%CARBON_HOME%" -Dlogfiles.home="%CARBON_HOME%\repository\logs" -Dwso2.server.standalone=true -Djava.command="%JAVA_HOME%\bin\java" -Djava.opts="%JAVA_OPTS%" -Djava.io.tmpdir="%CARBON_HOME%\tmp" -Dcatalina.base="%CARBON_HOME%\wso2\lib\tomcat" -Dwso2.carbon.xml="%CARBON_HOME%\conf\carbon.xml" -Dwso2.registry.xml="%CARBON_HOME%\conf\registry.xml" -Dwso2.user.mgt.xml="%CARBON_HOME%\conf\user-mgt.xml" -Dwso2.transports.xml="%CARBON_HOME%\conf\mgt-transports.xml" -Djava.util.logging.config.file="%CARBON_HOME%\conf\log4j.properties" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -DNonUserCoreMode=true -DNonRegistryMode=true -Dcarbon.logs.path="%CARBON_HOME%\repository\logs" -Dcomponents.repo="%CARBON_HOME%\wso2\components\plugins" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -Dcarbon.components.dir.path="%CARBON_HOME%\wso2\components" -Dcarbon.dropins.dir.path="%CARBON_HOME%\dropins" -Dcarbon.external.lib.dir.path="%CARBON_HOME%\lib" -Dcarbon.patches.dir.path="%CARBON_HOME%\patches" -Dcarbon.internal.lib.dir.path="%CARBON_HOME%\wso2\lib" -Dconf.location="%CARBON_HOME%\conf" -Dcom.atomikos.icatch.file="%CARBON_HOME%\wso2\lib\transactions.properties" -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler -Dcom.atomikos.icatch.hide_init_file_path="true" -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dcom.sun.jndi.ldap.connect.pool.authentication=simple -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 -Dorg.terracotta.quartz.skipUpdateCheck=true -Dcarbon.classpath=%CARBON_CLASSPATH% -Dfile.encoding=UTF8 -Dlogger.server.name="micro-integrator" -Dqpid.conf="\conf\advanced" -Dproperties.file.path=default -DavoidConfigHashRead=true -DenableReadinessProbe=true -DenableLivenessProbe=true -DenableManagementApi=true -DskipStartupExtensions=false -Dautomation.mode.seq.car.name="%CAR_NAME%" -Dlog4j2.contextSelector="org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" -Dorg.ops4j.pax.logging.logReaderEnabled=false -Dorg.ops4j.pax.logging.eventAdminEnabled=false %JAVA_VER_BASED_OPTS% %profile% -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" +set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof" -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% -Djava.endorsed.dirs=%JAVA_ENDORSED% -DandesConfig=broker.xml -Dcarbon.registry.root=/ -Dcarbon.home="%CARBON_HOME%" -Dlogfiles.home="%CARBON_HOME%\repository\logs" -Dwso2.server.standalone=true -Djava.command="%JAVA_HOME%\bin\java" -Djava.opts="%JAVA_OPTS%" -Djava.io.tmpdir="%CARBON_HOME%\tmp" -Dcatalina.base="%CARBON_HOME%\wso2\lib\tomcat" -Dwso2.carbon.xml="%CARBON_HOME%\conf\carbon.xml" -Dwso2.registry.xml="%CARBON_HOME%\conf\registry.xml" -Dwso2.user.mgt.xml="%CARBON_HOME%\conf\user-mgt.xml" -Dwso2.transports.xml="%CARBON_HOME%\conf\mgt-transports.xml" -Djava.util.logging.config.file="%CARBON_HOME%\conf\log4j.properties" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -DNonUserCoreMode=true -DNonRegistryMode=true -Dcarbon.logs.path="%CARBON_HOME%\repository\logs" -Dcomponents.repo="%CARBON_HOME%\wso2\components\plugins" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -Dcarbon.components.dir.path="%CARBON_HOME%\wso2\components" -Dcarbon.dropins.dir.path="%CARBON_HOME%\dropins" -Dcarbon.external.lib.dir.path="%CARBON_HOME%\lib" -Dcarbon.patches.dir.path="%CARBON_HOME%\patches" -Dcarbon.internal.lib.dir.path="%CARBON_HOME%\wso2\lib" -Dconf.location="%CARBON_HOME%\conf" -Dcom.atomikos.icatch.file="%CARBON_HOME%\wso2\lib\transactions.properties" -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler -Dcom.atomikos.icatch.hide_init_file_path="true" -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dcom.sun.jndi.ldap.connect.pool.authentication=simple -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 -Dorg.terracotta.quartz.skipUpdateCheck=true -Dcarbon.classpath=%CARBON_CLASSPATH% -Dfile.encoding=UTF8 -Dlogger.server.name="micro-integrator" -Dqpid.conf="\conf\advanced" -Dproperties.file.path=default -Djavax.xml.xpath.XPathFactory:http://java.sun.com/jaxp/xpath/dom=net.sf.saxon.xpath.XPathFactoryImpl -DavoidConfigHashRead=true -DenableReadinessProbe=true -DenableLivenessProbe=true -DenableManagementApi=true -DskipStartupExtensions=false -Dautomation.mode.seq.car.name="%CAR_NAME%" -Dlog4j2.contextSelector="org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" -Dorg.ops4j.pax.logging.logReaderEnabled=false -Dorg.ops4j.pax.logging.eventAdminEnabled=false %JAVA_VER_BASED_OPTS% %profile% -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" :runJava rem echo JAVA_HOME environment variable is set to %JAVA_HOME% From 7638c7daef3b0b5ac746b28e6da9c36e7f1ab1ef Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 18 Oct 2023 10:26:18 +0530 Subject: [PATCH 29/47] Upgrade snakeyaml This commit adds following changes 1. Upgarde snakeyaml to v2.2 2. Remove snakeyaml orbit dependency 3. Upgrade jakson dependencies to v2.15.2 4. Fixes for test failures --- .../pom.xml | 4 +- .../swagger/format/MediaTypeMixin.java | 66 +++++++++++++++++++ .../swagger/format/SwaggerGenerator.java | 56 +++++++++++++++- .../pom.xml | 4 +- pom.xml | 11 ++-- 5 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/MediaTypeMixin.java diff --git a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml index a796cabfbc..823f56d946 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml @@ -53,7 +53,7 @@ javax.servlet.http;version="${imp.pkg.version.javax.servlet}", org.apache.commons.logging; version="${import.package.version.commons.logging}", org.apache.axis2.*; version="${imp.pkg.version.axis2}", - org.yaml.snakeyaml.*; version="${orbit.snakeyaml.import.version.range}", + org.yaml.snakeyaml.*; version="${snakeyaml.import.version.range}", org.wso2.micro.core.*;version="${project.version}", *;resolution:=optional @@ -75,7 +75,7 @@ synapse-core - org.wso2.orbit.org.yaml + org.yaml snakeyaml diff --git a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/MediaTypeMixin.java b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/MediaTypeMixin.java new file mode 100644 index 0000000000..e345ab634b --- /dev/null +++ b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/MediaTypeMixin.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.micro.integrator.transport.handlers.requestprocessors.swagger.format; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.v3.oas.models.parameters.Parameter; + +import java.util.Set; + +public abstract class MediaTypeMixin { + public MediaTypeMixin() { + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public abstract Object getExample(); + + public void setExample(Object example) { + if (example != null) { + setExampleSetFlag(true); + } + } + + @JsonIgnore + abstract boolean getExampleSetFlag(); + + @JsonIgnore + public abstract void setExampleSetFlag(boolean exampleSetFlag); + + public abstract String getType(); + + @JsonIgnore + public abstract Set getTypes(); + + @JsonIgnore + public boolean shouldIgnoreTypes() { + return getType() != null; + } + + @JsonIgnore + public boolean shouldSerializeTypes() { + return getTypes() != null && !shouldIgnoreTypes(); + } + + @JsonIgnore + public abstract Parameter.StyleEnum getStyle(); + + @JsonIgnore + public abstract void setStyle(Parameter.StyleEnum style); +} diff --git a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/SwaggerGenerator.java b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/SwaggerGenerator.java index 3033274207..853c86b70e 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/SwaggerGenerator.java +++ b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/src/main/java/org/wso2/micro/integrator/transport/handlers/requestprocessors/swagger/format/SwaggerGenerator.java @@ -17,6 +17,18 @@ */ package org.wso2.micro.integrator.transport.handlers.requestprocessors.swagger.format; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.swagger.v3.core.util.Json; +import io.swagger.v3.core.util.Yaml; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.media.MediaType; +import io.swagger.v3.oas.models.media.Schema; +import io.swagger.v3.oas.models.parameters.Parameter; import org.apache.axiom.ext.io.StreamCopyException; import org.apache.axiom.util.blob.BlobOutputStream; import org.apache.axis2.AxisFault; @@ -52,8 +64,14 @@ public class SwaggerGenerator { */ protected void updateResponse(CarbonHttpResponse response, String responseString, String contentType) throws AxisFault { + String updatesResponseString = getOpenAPIJsonString(responseString, contentType); try { - byte[] responseStringBytes = responseString.getBytes(SwaggerConstants.DEFAULT_ENCODING); + if (updatesResponseString == null) { + // Parsing to OpenAPI model failed. Return the original response. + // This can happen when user save a custom swagger in registry. + updatesResponseString = responseString; + } + byte[] responseStringBytes = updatesResponseString.getBytes(SwaggerConstants.DEFAULT_ENCODING); ((BlobOutputStream) response.getOutputStream()).getBlob() .readFrom(new ByteArrayInputStream(responseStringBytes), responseStringBytes.length); } catch (StreamCopyException streamCopyException) { @@ -109,4 +127,40 @@ protected void handleException(String errorMsg) throws AxisFault { log.error(errorMsg); throw new AxisFault(errorMsg); } + + /** + * Ignore example:null from the response + * + * @param responseString String response to be updated in response + * @param contentType Content type of the response to be updated in response headers + */ + public static String getOpenAPIJsonString(String responseString, String contentType) { + if (contentType.contains(SwaggerConstants.CONTENT_TYPE_YAML)) { + try { + JsonNode yamlMapper = new ObjectMapper(new YAMLFactory()).readTree(responseString); + responseString = Json.mapper().writeValueAsString(yamlMapper); + } catch (JsonProcessingException e) { + log.error("Error while converting a yaml definition to a json " + e.getMessage()); + } + } + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + // this is to ignore "example" and "exampleSetFlag" objects when generating the OpenAPI + mapper.addMixIn(Parameter.class, MediaTypeMixin.class); + mapper.addMixIn(MediaType.class, MediaTypeMixin.class); + mapper.addMixIn(Schema.class, MediaTypeMixin.class); + try { + OpenAPI openAPI = mapper.readValue(responseString, OpenAPI.class); + String updatedResponseString = mapper.writeValueAsString(openAPI); + if (contentType.contains(SwaggerConstants.CONTENT_TYPE_JSON)) { + return updatedResponseString; + } + JsonNode jsonNodeTree = new ObjectMapper().readTree(updatedResponseString); + return Yaml.mapper().writeValueAsString(jsonNodeTree); + } catch (JsonProcessingException e) { + log.error("Error while generating Swagger JSON from model", e); + return null; + } + } } diff --git a/features/org.wso2.micro.integrator.initializer.feature/pom.xml b/features/org.wso2.micro.integrator.initializer.feature/pom.xml index 4098e2f2d2..0091de7083 100644 --- a/features/org.wso2.micro.integrator.initializer.feature/pom.xml +++ b/features/org.wso2.micro.integrator.initializer.feature/pom.xml @@ -55,7 +55,7 @@ zip - org.wso2.orbit.org.yaml + org.yaml snakeyaml @@ -96,7 +96,7 @@ org.wso2.ei:org.wso2.micro.integrator.registry:${project.version} - org.wso2.orbit.org.yaml:snakeyaml:${orbit.version.snakeyaml} + org.yaml:snakeyaml:${snakeyaml.version} org.apache.synapse.wso2:${synapse.version} diff --git a/pom.xml b/pom.xml index 83c0ef3a5f..53b1ce8b01 100644 --- a/pom.xml +++ b/pom.xml @@ -556,9 +556,9 @@ provided - org.wso2.orbit.org.yaml + org.yaml snakeyaml - ${orbit.version.snakeyaml} + ${snakeyaml.version} net.minidev @@ -1588,7 +1588,7 @@ 5.2.52 [5.2.0, 6.0.0) - 2.15.0 + 2.15.2 4.7.49 4.5.3 4.9.11 @@ -1672,7 +1672,6 @@ 2.3.2.wso2v1 1.2.4 4.2.0 - 2.0.0.wso2v1 2.4.11 31.0.1-jre 2.1.0.wso2v1 @@ -1693,7 +1692,7 @@ [0.0.0, 1.0.0) [2.6.0, 3.0.0) [1.2.0,2.0.0) - [2.0.0,3.0.0) + [2.0.0,3.0.0) [1.2.11, 1.3.0) [1.2.0,2.0.0) @@ -1868,7 +1867,7 @@ 2.0.30.wso2v1 1.6.5 - 2.0 + 2.2 1.1.0.Final 3.4.2 1.1 From aa9a5d9b82d3bd78c6f3ab332ee48350fe485d1d Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Wed, 18 Oct 2023 11:12:24 +0530 Subject: [PATCH 30/47] Fix dependency issue with apache poi Add stax2-api to classpath and introduced add-opens conditions to fix issues when using poi to read excel sheets Fixes /issues/2995 --- distribution/pom.xml | 4 ++++ distribution/src/assembly/bin.xml | 2 ++ distribution/src/scripts/micro-integrator.bat | 2 +- distribution/src/scripts/micro-integrator.sh | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 411a80bbf0..6b60eeb129 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -68,6 +68,10 @@ javax.jms javax.jms-api + + org.codehaus.woodstox + stax2-api + net.sf.saxon Saxon-HE diff --git a/distribution/src/assembly/bin.xml b/distribution/src/assembly/bin.xml index 6920477e5a..141f0af548 100755 --- a/distribution/src/assembly/bin.xml +++ b/distribution/src/assembly/bin.xml @@ -655,6 +655,8 @@ --> javax.xml.bind:jaxb-api:jar io.netty:netty-tcnative-boringssl-static + + org.codehaus.woodstox:stax2-api:jar diff --git a/distribution/src/scripts/micro-integrator.bat b/distribution/src/scripts/micro-integrator.bat index 4cfb26fe1e..83f02d299a 100644 --- a/distribution/src/scripts/micro-integrator.bat +++ b/distribution/src/scripts/micro-integrator.bat @@ -181,7 +181,7 @@ cd .. rem ---------- Add jars to classpath --c _CLASSPATH% if %JAVA_VERSION% GEQ 110 set CARBON_CLASSPATH=.\wso2\lib\*;%CARBON_CLASSPATH% -if %JAVA_VERSION% GEQ 110 set JAVA_VER_BASED_OPTS=--add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED +if %JAVA_VERSION% GEQ 110 set JAVA_VER_BASED_OPTS=--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.util=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED rem ---------------- Setting default profile for Runtime if not parsed -------------- diff --git a/distribution/src/scripts/micro-integrator.sh b/distribution/src/scripts/micro-integrator.sh index 4446577018..c48da08eab 100644 --- a/distribution/src/scripts/micro-integrator.sh +++ b/distribution/src/scripts/micro-integrator.sh @@ -280,7 +280,7 @@ fi JAVA_VER_BASED_OPTS="" if [ $java_version_formatted -ge 1100 ]; then - JAVA_VER_BASED_OPTS="--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED" + JAVA_VER_BASED_OPTS="--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.util=ALL-UNNAMED --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED" fi while [ "$status" = "$START_EXIT_STATUS" ] From 466ae5151661ce31c7814bf1ec00c3b37061677d Mon Sep 17 00:00:00 2001 From: chanikag Date: Tue, 24 Oct 2023 15:08:28 +0530 Subject: [PATCH 31/47] Upgrade synapse version $subject --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 35e5650d08..6cdf3aead8 100644 --- a/pom.xml +++ b/pom.xml @@ -1558,7 +1558,7 @@ 2.1.4.Final.wso2v1 - 4.0.0-wso2v47 + 4.0.0-wso2v57 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 2deb5473f43109eb3abb2fc6fa3f12ed5a859fec Mon Sep 17 00:00:00 2001 From: chanikag Date: Wed, 25 Oct 2023 11:31:49 +0530 Subject: [PATCH 32/47] Upgrade transport http version This is to upgrade transport http version which bundles upgraded snakeyaml version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6cdf3aead8..e61996725c 100644 --- a/pom.xml +++ b/pom.xml @@ -1574,7 +1574,7 @@ 1.1.15 2.0.0-wso2v67 - 6.3.42 + 6.3.49 3.0.3 1.3.0 From 2b7c0e39b85cbc9a726e039fa3932d70cd42bd3f Mon Sep 17 00:00:00 2001 From: chanikag Date: Thu, 26 Oct 2023 09:17:15 +0530 Subject: [PATCH 33/47] Upgrade axis2 version $subject --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e61996725c..894d2020aa 100644 --- a/pom.xml +++ b/pom.xml @@ -1581,9 +1581,9 @@ 2.1 - 1.6.1-wso2v94 + 1.6.1-wso2v96 - 1.6.1-wso2v64 + 1.6.1-wso2v96 [1.6.1, 1.7.0) 5.2.52 From 22ea525ee1d960f9a10ee572f2b34525f7b9c1a5 Mon Sep 17 00:00:00 2001 From: chanikag Date: Thu, 26 Oct 2023 10:20:50 +0530 Subject: [PATCH 34/47] Fixing ESBJAVA_4239_AccessHTTPSCAfterCallout Replacing Mocky EP with MI backend API --- .../default/api/HTTPSC500ReturnAPI.xml | 32 +++++++++++++++++++ .../default/endpoints/mockEP.xml | 2 +- .../default/api/HTTPSC500ReturnAPI.xml | 32 +++++++++++++++++++ .../default/endpoints/mockEP.xml | 2 +- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml create mode 100644 integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml diff --git a/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml b/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml new file mode 100644 index 0000000000..bdcacd322c --- /dev/null +++ b/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml @@ -0,0 +1,32 @@ + + + + + + + + + Internal Server Error + + + + + + + \ No newline at end of file diff --git a/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml b/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml index 14b601e651..1cc6353ee3 100644 --- a/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml +++ b/integration/mediation-tests/tests-patches-with-smb2/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml @@ -1,5 +1,5 @@ + uri-template="http://localhost:8480/HTTPSC500ReturnAPI"/> \ No newline at end of file diff --git a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml new file mode 100644 index 0000000000..bdcacd322c --- /dev/null +++ b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/api/HTTPSC500ReturnAPI.xml @@ -0,0 +1,32 @@ + + + + + + + + + Internal Server Error + + + + + + + \ No newline at end of file diff --git a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml index 14b601e651..1cc6353ee3 100644 --- a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml +++ b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/server/repository/deployment/server/synapse-configs/default/endpoints/mockEP.xml @@ -1,5 +1,5 @@ + uri-template="http://localhost:8480/HTTPSC500ReturnAPI"/> \ No newline at end of file From 4b022d072d0c83e5b526946ffefb53ded5638fce Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Wed, 25 Oct 2023 15:00:31 +0530 Subject: [PATCH 35/47] Improve the file-based user store Improve file-based user store to work as a primary user store. Fixes wso2/micro-integrator/issues/3006 --- .../http/api/ConfigurationLoader.java | 60 --- .../http/api/ConfigurationLoaderTestCase.java | 46 -- .../management/apis/ManagementApiParser.java | 16 - .../handler/AuthenticationHandlerAdapter.java | 1 + .../handler/AuthorizationHandler.java | 1 + .../handler/FileBasedUserStoreManager.java | 109 ---- .../handler/SecurityHandlerAdapter.java | 7 + .../apis/security/handler/SecurityUtils.java | 1 + .../apis/ManagementApiParserTest.java | 3 +- .../security/user/core/UserCoreConstants.java | 1 + .../core/common/AbstractUserStoreManager.java | 11 +- .../user/core/common/DefaultRealm.java | 30 +- .../user/core/common/DefaultRealmService.java | 18 +- .../security/user/core/dto/UserInfoDTO.java} | 11 +- .../core/file/FileBasedUserStoreManager.java | 499 ++++++++++++++++++ .../hybrid/FileBasedHybridRoleManager.java | 56 ++ .../user/core/jdbc/JDBCUserStoreManager.java | 52 +- .../ldap/ReadOnlyLDAPUserStoreManager.java | 33 +- .../ldap/ReadWriteLDAPUserStoreManager.java | 40 +- .../templates/conf/internal-apis.xml.j2 | 2 +- 20 files changed, 695 insertions(+), 302 deletions(-) delete mode 100644 components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/FileBasedUserStoreManager.java rename components/{mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/UserInfo.java => org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/dto/UserInfoDTO.java} (79%) create mode 100644 components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/file/FileBasedUserStoreManager.java create mode 100644 components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/hybrid/FileBasedHybridRoleManager.java diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/ConfigurationLoader.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/ConfigurationLoader.java index a73f4b5bd2..c42601da4d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/ConfigurationLoader.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/ConfigurationLoader.java @@ -63,12 +63,6 @@ public class ConfigurationLoader { private static final QName PROTOCOL_Q = new QName("protocol"); private static final QName HANDLERS_Q = new QName("handlers"); private static final QName RESOURCES_Q = new QName("resources"); - private static final QName USER_STORE_Q = new QName("userStore"); - private static final QName USERS_Q = new QName("users"); - private static final QName USER_Q = new QName("user"); - private static final QName USERNAME_Q = new QName("username"); - private static final QName PASSWORD_Q = new QName("password"); - private static final QName IS_ADMIN_Q = new QName("isAdmin"); private static final QName STORE_PASSWORD_Q = new QName("Password"); private static final QName KEY_PASSWORD_Q = new QName("KeyPassword"); @@ -84,7 +78,6 @@ public class ConfigurationLoader { private static SSLConfiguration sslConfiguration; private static boolean sslConfiguredSuccessfully; - private static Map userMap; private static List internalHttpApiList = new ArrayList<>(); private static List internalHttpsApiList = new ArrayList<>(); @@ -107,7 +100,6 @@ public static void loadInternalApis(String apiFilePath) { } setSecretResolver(apiConfig); - populateUserStore(apiConfig); Iterator apiIterator = apiConfig.getChildrenWithLocalName(APIS); @@ -187,54 +179,6 @@ public static void loadInternalApis(String apiFilePath) { } } - /** - * Populates the userList hashMap by userStore OM element - */ - private static void populateUserStore(OMElement apiConfig) { - OMElement userStoreOM = apiConfig.getFirstChildWithName(USER_STORE_Q); - if (Objects.nonNull(userStoreOM)) { - userMap = populateUsers(userStoreOM.getFirstChildWithName(USERS_Q)); - } else { - userMap = null; - } - } - - /** - * Populates individual users. - * - * @param users the parent element of users - * @return map of users against UserInfo config - */ - private static Map populateUsers(OMElement users) { - HashMap userMap = new HashMap<>(); - if (users != null) { - @SuppressWarnings("unchecked") - Iterator usersIterator = users.getChildrenWithName(USER_Q); - if (usersIterator != null) { - while (usersIterator.hasNext()) { - OMElement userElement = usersIterator.next(); - OMElement userNameElement = userElement.getFirstChildWithName(USERNAME_Q); - OMElement passwordElement = userElement.getFirstChildWithName(PASSWORD_Q); - OMElement isAdminElement = userElement.getFirstChildWithName(IS_ADMIN_Q); - if (userNameElement != null && passwordElement != null) { - String userName = userNameElement.getText(); - if (userMap.containsKey(userName)) { - handleException("Error parsing the file based user store. User: " + userName + " defined " - + "more than once. "); - } - boolean isAdmin = false; - if (isAdminElement != null) { - isAdmin = Boolean.parseBoolean(isAdminElement.getText().trim()); - } - userMap.put(userName, new UserInfo(userName, - resolveSecret(passwordElement.getText()).toCharArray(), isAdmin)); - } - } - } - } - return userMap; - } - /** * Checks if the text is protected and returns decrypted text if protected, else returns the plain text * @param text @@ -337,10 +281,6 @@ private static InternalAPI createApi(String classFQName) { } } - public static Map getUserMap() { - return userMap; - } - public static int getInternalInboundHttpPort() { return getPort(Constants.INTERNAL_HTTP_API_PORT, internalInboundHttpPortProperty, diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/java/internal/http/api/ConfigurationLoaderTestCase.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/java/internal/http/api/ConfigurationLoaderTestCase.java index 76d22b8ffe..de69fcc672 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/java/internal/http/api/ConfigurationLoaderTestCase.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/java/internal/http/api/ConfigurationLoaderTestCase.java @@ -24,7 +24,6 @@ import org.wso2.carbon.inbound.endpoint.internal.http.api.Constants; import org.wso2.carbon.inbound.endpoint.internal.http.api.InternalAPI; import org.wso2.carbon.inbound.endpoint.internal.http.api.InternalAPIHandler; -import org.wso2.carbon.inbound.endpoint.internal.http.api.UserInfo; import java.net.URL; import java.util.List; @@ -78,51 +77,6 @@ public void testLoadHandlers() { } - /** - * Test loading of internal apis from the internal-apis.xml file. - */ - @Test - public void testLoadUsers() { - - // test configuration with users - URL url = getClass().getResource("internal-apis.xml"); - Assert.assertNotNull("Configuration file not found", url); - - ConfigurationLoader.loadInternalApis("internal/http/api/internal-apis.xml"); - Map userMap = ConfigurationLoader.getUserMap(); - - org.junit.Assert.assertEquals(3, userMap.size()); - //Assert admin:admin - org.junit.Assert.assertNotNull(userMap.get("admin")); - org.junit.Assert.assertEquals("admin", String.valueOf(userMap.get("admin").getPassword())); - org.junit.Assert.assertTrue("isAdmin", userMap.get("admin").isAdmin()); - - //Assert user1:pwd1 - org.junit.Assert.assertNotNull(userMap.get("user1")); - org.junit.Assert.assertEquals("pwd1", String.valueOf(userMap.get("user1").getPassword())); - org.junit.Assert.assertFalse("isAdmin", userMap.get("user1").isAdmin()); - - //Assert user2:pwd2 - org.junit.Assert.assertNotNull(userMap.get("user2")); - org.junit.Assert.assertEquals("pwd2", String.valueOf(userMap.get("user2").getPassword())); - org.junit.Assert.assertFalse("isAdmin", userMap.get("user2").isAdmin()); - } - - /** - * Test loading of internal apis from the internal-apis.xml file. - */ - @Test - public void testLoadInternalApisWithNoUserStore() { - - // test configuration with users - URL url = getClass().getResource("internal-apis-without-user-store.xml"); - Assert.assertNotNull("Configuration file not found", url); - - ConfigurationLoader.loadInternalApis("internal/http/api/internal-apis-without-user-store.xml"); - Assert.assertNull("User store is not defined in the file but it is not null", - ConfigurationLoader.getUserMap()); - } - private List getApis() { System.setProperty(Constants.PREFIX_TO_ENABLE_INTERNAL_APIS + "SampleAPI", "true"); URL url = getClass().getResource("internal-apis.xml"); diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementApiParser.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementApiParser.java index f3f75eaaf4..5e51091a4a 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementApiParser.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementApiParser.java @@ -22,7 +22,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.wso2.carbon.inbound.endpoint.internal.http.api.ConfigurationLoader; -import org.wso2.carbon.inbound.endpoint.internal.http.api.UserInfo; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; import org.wso2.securevault.SecretResolverFactory; @@ -89,21 +88,6 @@ public static OMElement getManagementApiElement() throws IOException, CarbonExce throw new ManagementApiUndefinedException("Management API not defined in " + getConfigurationFilePath()); } - /** - * Method to get the user store define in internal-apis.xml - * - * @return a non null map if the user store is defined. - * @throws UserStoreUndefinedException if the user store is not defined in internal-apis.xml - */ - public Map getUserMap() throws UserStoreUndefinedException { - Map usersMap = ConfigurationLoader.getUserMap(); - if (Objects.nonNull(usersMap)) { - return usersMap; - } else { - throw new UserStoreUndefinedException("UserStore tag not defined inside the Management API"); - } - } - private static OMElement getInternalApisElement() throws IOException, CarbonException, XMLStreamException { File mgtApiUserConfig = getConfigurationFile(); try (InputStream fileInputStream = new FileInputStream(mgtApiUserConfig)) { diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthenticationHandlerAdapter.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthenticationHandlerAdapter.java index cbdd9d6125..5fd6de6ad5 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthenticationHandlerAdapter.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthenticationHandlerAdapter.java @@ -28,6 +28,7 @@ import org.wso2.micro.integrator.management.apis.ManagementApiUndefinedException; import org.wso2.micro.integrator.security.MicroIntegratorSecurityUtils; import org.wso2.micro.integrator.security.user.api.UserStoreException; +import org.wso2.micro.integrator.security.user.core.file.FileBasedUserStoreManager; import java.io.IOException; import java.util.Map; diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthorizationHandler.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthorizationHandler.java index a1556858a2..b91ab4d579 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthorizationHandler.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/AuthorizationHandler.java @@ -25,6 +25,7 @@ import org.wso2.micro.integrator.management.apis.ManagementApiUndefinedException; import org.wso2.micro.integrator.security.MicroIntegratorSecurityUtils; import org.wso2.micro.integrator.security.user.api.UserStoreException; +import org.wso2.micro.integrator.security.user.core.file.FileBasedUserStoreManager; import javax.xml.stream.XMLStreamException; import java.io.IOException; diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/FileBasedUserStoreManager.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/FileBasedUserStoreManager.java deleted file mode 100644 index 2612b5e7b0..0000000000 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/FileBasedUserStoreManager.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.wso2.micro.integrator.management.apis.security.handler; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.wso2.carbon.inbound.endpoint.internal.http.api.UserInfo; -import org.wso2.micro.integrator.management.apis.ManagementApiParser; -import org.wso2.micro.integrator.management.apis.UserStoreUndefinedException; - -import java.util.Map; - -/** - * This class is used to authenticate, authorize users against the File based user store defined in internal-apis.xml - */ -public class FileBasedUserStoreManager { - - private static final Log LOG = LogFactory.getLog(FileBasedUserStoreManager.class); - private static final FileBasedUserStoreManager userStoreManager = new FileBasedUserStoreManager(); - private static Map usersList; - private static boolean isInitialized = false; - - private FileBasedUserStoreManager() { - - initializeUserStore(); - } - - /** - * Method to retrieve FileBasedUserStoreManager - * - * @return FileBasedUserStoreManager - */ - public static FileBasedUserStoreManager getUserStoreManager() { - - return userStoreManager; - } - - /** - * Authenticate the user against the file based user store. - * - * @param username the user to be authenticated - * @param password the password used for authentication - * @return true if authenticated - */ - public boolean authenticate(String username, String password) { - - if (usersList.containsKey(username)) { - String passwordFromStore = String.valueOf(usersList.get(username).getPassword()); - if (StringUtils.isNotBlank(passwordFromStore) && passwordFromStore.equals(password)) { - return true; - } - } - return false; - } - - /** - * Method to assert if a user is an admin - * - * @param username the user to be validated as an admin - * @return true if the admin role is assigned to the user - */ - public boolean isAdmin(String username) { - - if (usersList.containsKey(username)) { - UserInfo userInfo = usersList.get(username); - return userInfo.isAdmin(); - } - return false; - } - - /** - * Method to check whether FileBasedUserStoreManager is initialized - * - * @return true if successfully initialized - */ - public boolean isInitialized() { - - return isInitialized; - } - - private static void initializeUserStore() { - - ManagementApiParser mgtParser = new ManagementApiParser(); - try { - usersList = mgtParser.getUserMap(); - isInitialized = true; - } catch (UserStoreUndefinedException e) { - LOG.error("User store config has not been defined in file " - + ManagementApiParser.getConfigurationFilePath(), e); - } - } -} diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityHandlerAdapter.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityHandlerAdapter.java index 6b4292ed5e..b878c09b3a 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityHandlerAdapter.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityHandlerAdapter.java @@ -23,9 +23,11 @@ import org.apache.synapse.MessageContext; import org.apache.synapse.rest.RESTConstants; import org.wso2.carbon.inbound.endpoint.internal.http.api.InternalAPIHandler; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.integrator.management.apis.Constants; import org.wso2.micro.integrator.management.apis.ManagementApiUndefinedException; +import org.wso2.micro.integrator.security.user.core.file.FileBasedUserStoreManager; import java.io.IOException; import java.util.ArrayList; @@ -40,6 +42,7 @@ public abstract class SecurityHandlerAdapter implements InternalAPIHandler { protected static boolean useCarbonUserStore = false; private static boolean isInitialized = false; + private static String FILE_BASED_USER_STORE_AS_PRIMARY = "internal_apis.file_user_store.primary"; /** * Resources defined in internal-apis.xml to be handled */ @@ -67,6 +70,10 @@ private static void initializeUserStore() throws CarbonException, IOException, M XMLStreamException { if (!isInitialized) { if (SecurityUtils.isFileBasedUserStoreEnabled()) { + Object fileUserStore = ConfigParser.getParsedConfigs().get(FILE_BASED_USER_STORE_AS_PRIMARY); + if (fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString())) { + useCarbonUserStore = true; + } isInitialized = FileBasedUserStoreManager.getUserStoreManager().isInitialized(); } else { LOG.info("File based user store has been disabled. Carbon user store settings will be used."); diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityUtils.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityUtils.java index 9c8cc10b8f..ef00e46a5c 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityUtils.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/security/handler/SecurityUtils.java @@ -27,6 +27,7 @@ import org.wso2.micro.integrator.management.apis.Constants; import org.wso2.micro.integrator.security.MicroIntegratorSecurityUtils; import org.wso2.micro.integrator.security.user.api.UserStoreException; +import org.wso2.micro.integrator.security.user.core.file.FileBasedUserStoreManager; import org.wso2.securevault.SecretResolver; import org.wso2.securevault.SecureVaultException; diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/test/java/org/wso2/micro/integrator/management/apis/ManagementApiParserTest.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/test/java/org/wso2/micro/integrator/management/apis/ManagementApiParserTest.java index 629b64e9ff..17b3bd930a 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/test/java/org/wso2/micro/integrator/management/apis/ManagementApiParserTest.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/test/java/org/wso2/micro/integrator/management/apis/ManagementApiParserTest.java @@ -28,7 +28,6 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.wso2.carbon.inbound.endpoint.internal.http.api.ConfigurationLoader; -import org.wso2.carbon.inbound.endpoint.internal.http.api.UserInfo; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.integrator.core.internal.MicroIntegratorBaseConstants; @@ -60,6 +59,7 @@ public void getManagementApiElement() throws CarbonException, XMLStreamException Assert.assertEquals(MGT_API_NAME, managementApiElement.getAttributeValue(NAME_ATTR)); } + /* @Test public void testGetUserList() throws UserStoreUndefinedException { Map userMap = new HashMap<>(); @@ -79,4 +79,5 @@ public void testGetNullUserStore() throws UserStoreUndefinedException { ManagementApiParser managementApiParser = new ManagementApiParser(); managementApiParser.getUserMap(); } + */ } diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/UserCoreConstants.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/UserCoreConstants.java index cfc558ec6d..696f28e681 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/UserCoreConstants.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/UserCoreConstants.java @@ -67,6 +67,7 @@ public class UserCoreConstants { public static final String IS_USER_IN_ROLE_CACHE_IDENTIFIER = "@__isUserHasTheRole__@"; public static final String DOMAIN_SEPARATOR; + public static final String FILE_BASED_USER_STORE_AS_PRIMARY = "internal_apis.file_user_store.primary"; static { String userDomainSeparator = CarbonServerConfigurationService.getInstance().getFirstProperty("UserDomainSeparator"); diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/AbstractUserStoreManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/AbstractUserStoreManager.java index 6e56da6d13..cf3413dcc0 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/AbstractUserStoreManager.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/AbstractUserStoreManager.java @@ -40,6 +40,7 @@ import org.wso2.micro.integrator.security.user.core.constants.UserCoreErrorConstants; import org.wso2.micro.integrator.security.user.core.constants.UserCoreErrorConstants.ErrorMessages; import org.wso2.micro.integrator.security.user.core.dto.RoleDTO; +import org.wso2.micro.integrator.security.user.core.hybrid.FileBasedHybridRoleManager; import org.wso2.micro.integrator.security.user.core.hybrid.HybridRoleManager; import org.wso2.micro.integrator.security.user.core.hybrid.JdbcHybridRoleManager; import org.wso2.micro.integrator.security.user.core.internal.UMListenerServiceComponent; @@ -5790,9 +5791,13 @@ protected List getMappingAttributeList(List claimList) return attributeList; } - protected void doInitialSetup() throws UserStoreException { - systemUserRoleManager = new SystemUserRoleManager(dataSource, tenantId); - hybridRoleManager = new JdbcHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); + protected void doInitialSetup(boolean isFileBased) throws UserStoreException { + if (!isFileBased) { + systemUserRoleManager = new SystemUserRoleManager(dataSource, tenantId); + hybridRoleManager = new JdbcHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); + } else { + hybridRoleManager = new FileBasedHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); + } } /** diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealm.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealm.java index 3ed8b7ae91..85b2b694c6 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealm.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealm.java @@ -19,6 +19,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.snmp4j.User; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.integrator.security.user.api.RealmConfiguration; import org.wso2.micro.integrator.security.user.core.AuthorizationManager; import org.wso2.micro.integrator.security.user.core.UserCoreConstants; @@ -62,6 +64,8 @@ public class DefaultRealm implements UserRealm { private AuthorizationManager authzManager = null; private Map properties = null; + private boolean fileBasedUserStoreMode = false; + /** * Usage of this method is found on tests. * @@ -115,16 +119,21 @@ public void init(RealmConfiguration configBean, Map propertiesMa Map claimMappings = new HashMap(); Map profileConfigs = new HashMap(); - - if (Boolean.parseBoolean(realmConfig.getRealmProperty(UserCoreClaimConstants.INITIALIZE_NEW_CLAIM_MANAGER))) { - if (UserStoreMgtDSComponent.getClaimManagerFactory() != null) { - claimMan = UserStoreMgtDSComponent.getClaimManagerFactory().createClaimManager(tenantId); + Object fileUserStore = ConfigParser.getParsedConfigs().get(UserCoreConstants.FILE_BASED_USER_STORE_AS_PRIMARY); + if (fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString())) { + fileBasedUserStoreMode = true; + } + if (!fileBasedUserStoreMode) { + if (Boolean.parseBoolean(realmConfig.getRealmProperty(UserCoreClaimConstants.INITIALIZE_NEW_CLAIM_MANAGER))) { + if (UserStoreMgtDSComponent.getClaimManagerFactory() != null) { + claimMan = UserStoreMgtDSComponent.getClaimManagerFactory().createClaimManager(tenantId); + } else { + claimMan = new InMemoryClaimManager(); + } } else { - claimMan = new InMemoryClaimManager(); + populateProfileAndClaimMaps(claimMappings, profileConfigs); + claimMan = new DefaultClaimManager(claimMappings, dataSource, tenantId); } - } else { - populateProfileAndClaimMaps(claimMappings, profileConfigs); - claimMan = new DefaultClaimManager(claimMappings, dataSource, tenantId); } initializeObjects(); } @@ -310,8 +319,9 @@ private void initializeObjects() throws UserStoreException { log.error(message); throw new UserStoreException(message); } - this.authzManager = (AuthorizationManager) createObjectWithOptions(value, realmConfig, - properties); + if (!fileBasedUserStoreMode) { + this.authzManager = (AuthorizationManager) createObjectWithOptions(value, realmConfig, properties); + } } catch (Exception e) { log.error(e.getMessage(), e); diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealmService.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealmService.java index b485757f20..46d6432973 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealmService.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/common/DefaultRealmService.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.osgi.framework.BundleContext; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.core.Constants; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.integrator.core.internal.MicroIntegratorBaseConstants; @@ -83,13 +84,16 @@ public DefaultRealmService(BundleContext bc, RealmConfiguration realmConfig) thr } else { this.bootstrapRealmConfig = buildBootStrapRealmConfig(); } -// this.tenantMgtConfiguration = buildTenantMgtConfig(bc, -// this.bootstrapRealmConfig.getUserStoreProperty(UserCoreConstants.TenantMgtConfig.LOCAL_NAME_TENANT_MANAGER)); - this.dataSource = DatabaseUtil.getRealmDataSource(bootstrapRealmConfig); - // TODO We do not need to init DB for now - //initializeDatabase(dataSource); - properties.put(UserCoreConstants.DATA_SOURCE, dataSource); - properties.put(UserCoreConstants.FIRST_STARTUP_CHECK, isFirstInitialization); + Object fileUserStore = ConfigParser.getParsedConfigs().get(UserCoreConstants.FILE_BASED_USER_STORE_AS_PRIMARY); + if (!(fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString()))) { + // this.tenantMgtConfiguration = buildTenantMgtConfig(bc, + // this.bootstrapRealmConfig.getUserStoreProperty(UserCoreConstants.TenantMgtConfig.LOCAL_NAME_TENANT_MANAGER)); + this.dataSource = DatabaseUtil.getRealmDataSource(bootstrapRealmConfig); + // TODO We do not need to init DB for now + //initializeDatabase(dataSource); + properties.put(UserCoreConstants.DATA_SOURCE, dataSource); + properties.put(UserCoreConstants.FIRST_STARTUP_CHECK, isFirstInitialization); + } //this.tenantManager = this.initializeTenantManger(this.getTenantConfigurationElement(bc)); // this.tenantManager = this.initializeTenantManger(this.tenantMgtConfiguration); diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/UserInfo.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/dto/UserInfoDTO.java similarity index 79% rename from components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/UserInfo.java rename to components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/dto/UserInfoDTO.java index cd07453c58..9d34b47dd5 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/internal/http/api/UserInfo.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/dto/UserInfoDTO.java @@ -1,7 +1,7 @@ /* - * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. * - * WSO2 Inc. licenses this file to you under the Apache License, + * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except * in compliance with the License. * You may obtain a copy of the License at @@ -16,18 +16,17 @@ * under the License. */ -package org.wso2.carbon.inbound.endpoint.internal.http.api; +package org.wso2.micro.integrator.security.user.core.dto; /** * This class is the DTO for User configs in internal-apis.xml */ -public class UserInfo { - +public class UserInfoDTO { private String username; private char[] password; private boolean admin; - public UserInfo(String username, char[] password, boolean admin) { + public UserInfoDTO(String username, char[] password, boolean admin) { this.username = username; this.password = password; diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/file/FileBasedUserStoreManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/file/FileBasedUserStoreManager.java new file mode 100644 index 0000000000..7c386f82dd --- /dev/null +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/file/FileBasedUserStoreManager.java @@ -0,0 +1,499 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.micro.integrator.security.user.core.file; + +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.impl.builder.StAXOMBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.wso2.micro.core.util.CarbonException; +import org.wso2.micro.integrator.core.util.MicroIntegratorBaseUtils; +import org.wso2.micro.integrator.security.UnsupportedSecretTypeException; +import org.wso2.micro.integrator.security.user.api.Properties; +import org.wso2.micro.integrator.security.user.api.RealmConfiguration; +import org.wso2.micro.integrator.security.user.api.Tenant; +import org.wso2.micro.integrator.security.user.api.UserStoreException; +import org.wso2.micro.integrator.security.user.core.UserCoreConstants; +import org.wso2.micro.integrator.security.user.core.UserRealm; +import org.wso2.micro.integrator.security.user.core.claim.ClaimManager; +import org.wso2.micro.integrator.security.user.core.common.AbstractUserStoreManager; +import org.wso2.micro.integrator.security.user.core.common.RoleContext; +import org.wso2.micro.integrator.security.user.core.dto.UserInfoDTO; +import org.wso2.micro.integrator.security.user.core.profile.ProfileConfigurationManager; +import org.wso2.micro.integrator.security.user.core.util.JDBCRealmUtil; +import org.wso2.micro.integrator.security.util.Secret; +import org.wso2.securevault.SecretResolver; +import org.wso2.securevault.SecretResolverFactory; +import org.wso2.securevault.commons.MiscellaneousUtil; + +import javax.xml.namespace.QName; +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; + +/** + * This class is used to authenticate, authorize users against the File based user store defined in internal-apis.xml + */ +public class FileBasedUserStoreManager extends AbstractUserStoreManager { + + private static final Log log = LogFactory.getLog(FileBasedUserStoreManager.class); + private static final FileBasedUserStoreManager userStoreManager = new FileBasedUserStoreManager(); + private static Map usersList; + private static boolean isInitialized = false; + private static final String USER_STORE = "userStore"; + private static final String USERS = "users"; + private static final String USER = "user"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + private static final String IS_ADMIN = "isAdmin"; + private static SecretResolver secretResolver; + + private FileBasedUserStoreManager() { + initializeUserStore(); + } + + /** + * Method to retrieve FileBasedUserStoreManager + * + * @return FileBasedUserStoreManager + */ + public static FileBasedUserStoreManager getUserStoreManager() { + return userStoreManager; + } + + /** + * Authenticate the user against the file based user store. + * + * @param username the user to be authenticated + * @param password the password used for authentication + * @return true if authenticated + */ + public boolean authenticate(String username, String password) { + if (usersList.containsKey(username)) { + String passwordFromStore = String.valueOf(usersList.get(username).getPassword()); + return StringUtils.isNotBlank(passwordFromStore) && passwordFromStore.equals(password); + } + return false; + } + + /** + * Method to assert if a user is an admin + * + * @param username the user to be validated as an admin + * @return true if the admin role is assigned to the user + */ + public boolean isAdmin(String username) { + if (usersList.containsKey(username)) { + UserInfoDTO userInfo = usersList.get(username); + return userInfo.isAdmin(); + } + return false; + } + + /** + * Method to check whether a user exists in the FileBasedUserStoreManager. + * + * @param username the user to be checked. + * @return true if the user exists. + */ + public boolean isUserExists(String username) { + return usersList.containsKey(username); + } + + /** + * Method to check whether FileBasedUserStoreManager is initialized + * + * @return true if successfully initialized + */ + public boolean isInitialized() { + return isInitialized; + } + + /** + * Method to initialize FileBasedUserStoreManager using the internal-apis.xml + */ + private static void initializeUserStore() { + OMElement documentElement = null; + // Fetch users from internal-apis.xml + File mgtApiUserConfig = new File(MicroIntegratorBaseUtils.getCarbonConfigDirPath(), "internal-apis.xml"); + try (InputStream fileInputStream = Files.newInputStream(mgtApiUserConfig.toPath())) { + InputStream inputStream = MicroIntegratorBaseUtils.replaceSystemVariablesInXml(fileInputStream); + StAXOMBuilder builder = new StAXOMBuilder(inputStream); + documentElement = builder.getDocumentElement(); + } catch (IOException | CarbonException | XMLStreamException e) { + log.error("Error parsing the file based user store. Error reading internal-apis.xml", e); + } + if (documentElement == null) { + log.error("Error parsing the file based user store. Error reading internal-apis.xml"); + return; + } + secretResolver = SecretResolverFactory.create(documentElement, true); + OMElement userStoreOM = (OMElement) documentElement.getChildrenWithName(new QName(USER_STORE)).next(); + if (Objects.nonNull(userStoreOM)) { + usersList = populateUsers(userStoreOM.getFirstChildWithName(new QName(USERS))); + isInitialized = true; + } else { + log.error("Error parsing the file based user store. User store element not found in internal-apis.xml"); + } + } + + @Override + public Map getProperties(Tenant tenant) throws UserStoreException { + return null; + } + + @Override + public boolean isMultipleProfilesAllowed() { + return false; + } + + @Override + public void addRememberMe(String userName, String token) throws UserStoreException { + + } + + @Override + public boolean isValidRememberMeToken(String userName, String token) throws UserStoreException { + return false; + } + + @Override + public Properties getDefaultUserStoreProperties() { + return null; + } + + @Override + public String[] getProfileNames(String userName) { + return new String[0]; + } + + @Override + public String[] getAllProfileNames() { + return new String[0]; + } + + @Override + public boolean isReadOnly() throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return false; + } + + @Override + public int getUserId(String username) { + return 0; + } + + @Override + public int getTenantId(String username) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return 0; + } + + @Override + public int getTenantId() throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return 0; + } + + @Override + public Map getProperties(org.wso2.micro.integrator.security.user.core.tenant.Tenant tenant) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return null; + } + + @Override + public boolean isBulkImportSupported() { + return false; + } + + @Override + public RealmConfiguration getRealmConfiguration() { + return this.realmConfig; + } + + @Override + protected Map getUserPropertyValues(String userName, String[] propertyNames, String profileName) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return null; + } + + @Override + protected boolean doCheckExistingRole(String roleName) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return false; + } + + @Override + protected RoleContext createRoleContext(String roleName) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return null; + } + + @Override + protected boolean doCheckExistingUser(String userName) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return false; + } + + @Override + protected String[] getUserListFromProperties(String property, String value, String profileName) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + @Override + protected boolean doAuthenticate(String userName, Object credential) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + if (credential == null || ((Secret) credential).getChars().length == 0) { + return false; + } else { + try { + Secret password = Secret.getSecret(credential); + return authenticate(userName, new String(password.getChars())); + } catch (UnsupportedSecretTypeException e) { + return false; + } + } + } + + @Override + protected void doAddUser(String userName, Object credential, String[] roleList, Map claims, String profileName, boolean requirePasswordChange) throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected void doUpdateCredential(String userName, Object newCredential, Object oldCredential) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doUpdateCredentialByAdmin(String userName, Object newCredential) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doDeleteUser(String userName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doSetUserClaimValue(String userName, String claimURI, String claimValue, String profileName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected void doSetUserClaimValues(String userName, Map claims, String profileName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doDeleteUserClaimValue(String userName, String claimURI, String profileName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doDeleteUserClaimValues(String userName, String[] claims, String profileName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + } + + @Override + protected void doUpdateUserListOfRole(String roleName, String[] deletedUsers, String[] newUsers) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected void doUpdateRoleListOfUser(String userName, String[] deletedRoles, String[] newRoles) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected String[] doGetExternalRoleListOfUser(String userName, String filter) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + @Override + protected String[] doGetSharedRoleListOfUser(String userName, String tenantDomain, String filter) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + @Override + protected void doAddRole(String roleName, String[] userList, boolean shared) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected void doDeleteRole(String roleName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected void doUpdateRoleName(String roleName, String newRoleName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + + } + + @Override + protected String[] doGetRoleNames(String filter, int maxItemLimit) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + // admin is the only role supported in file based user store + return new String[]{"admin"}; + } + + @Override + protected String[] doListUsers(String filter, int maxItemLimit) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return usersList.keySet().toArray(new String[0]); + } + + @Override + protected String[] doGetDisplayNamesForInternalRole(String[] userNames) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + @Override + public boolean doCheckIsUserInRole(String userName, String roleName) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return false; + } + + @Override + protected String[] doGetSharedRoleNames(String tenantDomain, String filter, int maxItemLimit) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + @Override + protected String[] doGetUserListOfRole(String roleName, String filter) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + return new String[0]; + } + + + public FileBasedUserStoreManager(RealmConfiguration realmConfig, Map properties, + ClaimManager claimManager, ProfileConfigurationManager profileManager, + UserRealm realm, Integer tenantId) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + this(realmConfig, tenantId); + this.claimManager = claimManager; + this.userRealm = realm; + doInitialSetup(true); + } + + public FileBasedUserStoreManager(RealmConfiguration realmConfig, int tenantId) + throws org.wso2.micro.integrator.security.user.core.UserStoreException { + this.realmConfig = realmConfig; + this.tenantId = tenantId; + realmConfig.setUserStoreProperties(JDBCRealmUtil.getSQL(realmConfig + .getUserStoreProperties())); + + // new properties after carbon core 4.0.7 release. + if (realmConfig.getUserStoreProperty(UserCoreConstants.RealmConfig.READ_GROUPS_ENABLED) != null) { + readGroupsEnabled = Boolean.parseBoolean(realmConfig + .getUserStoreProperty(UserCoreConstants.RealmConfig.READ_GROUPS_ENABLED)); + } + + if (realmConfig.getUserStoreProperty(UserCoreConstants.RealmConfig.WRITE_GROUPS_ENABLED) != null) { + writeGroupsEnabled = Boolean.parseBoolean(realmConfig + .getUserStoreProperty(UserCoreConstants.RealmConfig.WRITE_GROUPS_ENABLED)); + } else { + if (!isReadOnly()) { + writeGroupsEnabled = true; + } + } + + // This property is now deprecated + if (realmConfig + .getUserStoreProperty(UserCoreConstants.RealmConfig.PROPERTY_INTERNAL_ROLES_ONLY) != null) { + boolean internalRolesOnly = Boolean + .parseBoolean(realmConfig + .getUserStoreProperty(UserCoreConstants.RealmConfig.PROPERTY_INTERNAL_ROLES_ONLY)); + if (internalRolesOnly) { + readGroupsEnabled = false; + writeGroupsEnabled = false; + } else { + readGroupsEnabled = true; + writeGroupsEnabled = true; + } + } + + if (writeGroupsEnabled) { + readGroupsEnabled = true; + } + + } + + /** + * Populates individual users. + * + * @param users the parent element of users + * @return map of users against UserInfo config + */ + private static Map populateUsers(OMElement users) { + HashMap userMap = new HashMap<>(); + if (users != null) { + @SuppressWarnings("unchecked") + Iterator usersIterator = users.getChildrenWithName(new QName(USER)); + if (usersIterator != null) { + while (usersIterator.hasNext()) { + OMElement userElement = usersIterator.next(); + OMElement userNameElement = userElement.getFirstChildWithName(new QName(USERNAME)); + OMElement passwordElement = userElement.getFirstChildWithName(new QName(PASSWORD)); + OMElement isAdminElement = userElement.getFirstChildWithName(new QName(IS_ADMIN)); + if (userNameElement != null && passwordElement != null) { + String userName = userNameElement.getText(); + if (userMap.containsKey(userName)) { + System.out.println("Error parsing the file based user store. User: " + userName + " defined " + + "more than once."); + } + boolean isAdmin = false; + if (isAdminElement != null) { + isAdmin = Boolean.parseBoolean(isAdminElement.getText().trim()); + } + userMap.put(userName, new UserInfoDTO(userName, + resolveSecret(passwordElement.getText()).toCharArray(), isAdmin)); + } + } + } + } + return userMap; + } + + /** + * Checks if the text is protected and returns decrypted text if protected, else returns the plain text + * + * @param text text to be resolved + * @return Decrypted text if protected else plain text + */ + private static String resolveSecret(String text) { + String alias = MiscellaneousUtil.getProtectedToken(text); + if (!StringUtils.isEmpty(alias)) { + if (secretResolver.isInitialized()) { + return MiscellaneousUtil.resolve(alias, secretResolver); + } + } + return text; + } +} diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/hybrid/FileBasedHybridRoleManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/hybrid/FileBasedHybridRoleManager.java new file mode 100644 index 0000000000..9e223ba539 --- /dev/null +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/hybrid/FileBasedHybridRoleManager.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.micro.integrator.security.user.core.hybrid; + +import org.wso2.micro.integrator.security.user.api.RealmConfiguration; +import org.wso2.micro.integrator.security.user.core.UserRealm; +import org.wso2.micro.integrator.security.user.core.UserStoreException; +import org.wso2.micro.integrator.security.user.core.file.FileBasedUserStoreManager; + +import javax.sql.DataSource; + +/** + * RoleManager implementation for the file based user store. + */ +public class FileBasedHybridRoleManager extends HybridRoleManager { + public FileBasedHybridRoleManager(DataSource dataSource, int tenantId, RealmConfiguration realmConfig, + UserRealm realm) { + super(dataSource, tenantId, realmConfig, realm); + } + + /** + * Get the list of roles of a user + * + * @param userName user name + * @param filter filter + * @return list of roles + * @throws UserStoreException if an error occurs + */ + public String[] getHybridRoleListOfUser(String userName, String filter) throws UserStoreException { + FileBasedUserStoreManager userStoreManager = FileBasedUserStoreManager.getUserStoreManager(); + if (userStoreManager.isUserExists(userName)) { + if (userStoreManager.isAdmin(userName)) { + return new String[]{"admin", "Internal/everyone"}; + } else { + return new String[]{"Internal/everyone"}; + } + } else { + return new String[0]; + } + } +} diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/jdbc/JDBCUserStoreManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/jdbc/JDBCUserStoreManager.java index eed623fbac..15612cf905 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/jdbc/JDBCUserStoreManager.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/jdbc/JDBCUserStoreManager.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.snmp4j.User; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.core.Constants; import org.wso2.micro.core.util.DatabaseCreator; import org.wso2.micro.integrator.security.UnsupportedSecretTypeException; @@ -37,6 +38,7 @@ import org.wso2.micro.integrator.security.user.core.common.PaginatedSearchResult; import org.wso2.micro.integrator.security.user.core.common.RoleContext; import org.wso2.micro.integrator.security.user.core.dto.RoleDTO; +import org.wso2.micro.integrator.security.user.core.hybrid.FileBasedHybridRoleManager; import org.wso2.micro.integrator.security.user.core.hybrid.HybridJDBCConstants; import org.wso2.micro.integrator.security.user.core.jdbc.caseinsensitive.JDBCCaseInsensitiveConstants; import org.wso2.micro.integrator.security.user.core.model.Condition; @@ -99,11 +101,17 @@ public class JDBCUserStoreManager extends AbstractUserStoreManager { private static final String MSSQL = "mssql"; private static final String ORACLE = "oracle"; private static final String MYSQL = "mysql"; - + private boolean fileBasedUserStoreMode = false; public JDBCUserStoreManager() { } + private void checkFileBasedUserStoreEnabled() throws UserStoreException { + Object fileUserStore = ConfigParser.getParsedConfigs().get(UserCoreConstants.FILE_BASED_USER_STORE_AS_PRIMARY); + if (fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString())) { + fileBasedUserStoreMode = true; + } + } /** * @param realmConfig * @param tenantId @@ -194,14 +202,17 @@ public JDBCUserStoreManager(DataSource ds, RealmConfiguration realmConfig, int t this.jdbcds = ds; this.dataSource = ds; - if (dataSource == null) { - dataSource = DatabaseUtil.getRealmDataSource(realmConfig); - } - if (dataSource == null) { - throw new UserStoreException("User Management Data Source is null"); + checkFileBasedUserStoreEnabled(); + if (!fileBasedUserStoreMode) { + if (dataSource == null) { + dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + } + if (dataSource == null) { + throw new UserStoreException("User Management Data Source is null"); + } + this.persistDomain(); } - doInitialSetup(); - this.persistDomain(); + doInitialSetup(fileBasedUserStoreMode); if (addInitData && realmConfig.isPrimary()) { addInitialAdminData(Boolean.parseBoolean(realmConfig.getAddAdmin()), !isInitSetupDone()); @@ -284,22 +295,25 @@ public JDBCUserStoreManager(RealmConfiguration realmConfig, Map } } - dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); - if (dataSource == null) { - dataSource = DatabaseUtil.getRealmDataSource(realmConfig); - } - if (dataSource == null) { - throw new UserStoreException("User Management Data Source is null"); + checkFileBasedUserStoreEnabled(); + if (!fileBasedUserStoreMode) { + dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); + if (dataSource == null) { + dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + } + if (dataSource == null) { + throw new UserStoreException("User Management Data Source is null"); + } + properties.put(UserCoreConstants.DATA_SOURCE, dataSource); } - properties.put(UserCoreConstants.DATA_SOURCE, dataSource); - - realmConfig.setUserStoreProperties(JDBCRealmUtil.getSQL(realmConfig .getUserStoreProperties())); - this.persistDomain(); - doInitialSetup(); + if (!fileBasedUserStoreMode) { + this.persistDomain(); + } + doInitialSetup(fileBasedUserStoreMode); if (!skipInitData && realmConfig.isPrimary()) { addInitialAdminData(Boolean.parseBoolean(realmConfig.getAddAdmin()), !isInitSetupDone()); diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadOnlyLDAPUserStoreManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadOnlyLDAPUserStoreManager.java index 234f19fcff..0706e5ceb3 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadOnlyLDAPUserStoreManager.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadOnlyLDAPUserStoreManager.java @@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.core.Constants; import org.wso2.micro.integrator.security.UnsupportedSecretTypeException; import org.wso2.micro.integrator.security.user.core.UserCoreConstants; @@ -154,6 +155,8 @@ public class ReadOnlyLDAPUserStoreManager extends AbstractUserStoreManager { */ protected boolean emptyRolesAllowed = false; + private boolean fileBasedUserStoreMode = false; + static { setAdvancedProperties(); } @@ -202,17 +205,22 @@ public ReadOnlyLDAPUserStoreManager(RealmConfiguration realmConfig, // check if required configurations are in the user-mgt.xml checkRequiredUserStoreConfigurations(); - - dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); - if (dataSource == null) { - // avoid returning null - dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + Object fileUserStore = ConfigParser.getParsedConfigs().get(UserCoreConstants.FILE_BASED_USER_STORE_AS_PRIMARY); + if (fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString())) { + fileBasedUserStoreMode = true; } - if (dataSource == null) { - throw new UserStoreException("Data Source is null"); - } - properties.put(UserCoreConstants.DATA_SOURCE, dataSource); + if(!fileBasedUserStoreMode) { + dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); + if (dataSource == null) { + // avoid returning null + dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + } + if (dataSource == null) { + throw new UserStoreException("Data Source is null"); + } + properties.put(UserCoreConstants.DATA_SOURCE, dataSource); + } /* * obtain the ldap connection source that was created in * DefaultRealmService. @@ -234,8 +242,11 @@ public ReadOnlyLDAPUserStoreManager(RealmConfiguration realmConfig, JNDIUtil.closeContext(dirContext); } this.userRealm = realm; - this.persistDomain(); - doInitialSetup(); + + if (!fileBasedUserStoreMode) { + this.persistDomain(); + } + doInitialSetup(fileBasedUserStoreMode); if (realmConfig.isPrimary()) { addInitialAdminData(Boolean.parseBoolean(realmConfig.getAddAdmin()), !isInitSetupDone()); diff --git a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadWriteLDAPUserStoreManager.java b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadWriteLDAPUserStoreManager.java index d28a938e06..f2880248ba 100644 --- a/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadWriteLDAPUserStoreManager.java +++ b/components/org.wso2.micro.integrator.security/src/main/java/org/wso2/micro/integrator/security/user/core/ldap/ReadWriteLDAPUserStoreManager.java @@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.core.Constants; import org.wso2.micro.integrator.security.user.core.UserCoreConstants; import org.wso2.micro.integrator.security.user.api.Properties; @@ -30,6 +31,7 @@ import org.wso2.micro.integrator.security.user.core.UserStoreException; import org.wso2.micro.integrator.security.user.core.claim.ClaimManager; import org.wso2.micro.integrator.security.user.core.common.RoleContext; +import org.wso2.micro.integrator.security.user.core.hybrid.FileBasedHybridRoleManager; import org.wso2.micro.integrator.security.user.core.hybrid.HybridRoleManager; import org.wso2.micro.integrator.security.user.core.hybrid.JdbcHybridRoleManager; import org.wso2.micro.integrator.security.user.core.profile.ProfileConfigurationManager; @@ -102,6 +104,7 @@ public class ReadWriteLDAPUserStoreManager extends ReadOnlyLDAPUserStoreManager protected Random random = new Random(); protected boolean kdcEnabled = false; + private boolean fileBasedUserStoreMode = false; static { setAdvancedProperties(); @@ -130,22 +133,31 @@ public ReadWriteLDAPUserStoreManager(RealmConfiguration realmConfig, this.kdcEnabled = UserCoreUtil.isKdcEnabled(realmConfig); checkRequiredUserStoreConfigurations(); - - dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); - if (dataSource == null) { - // avoid returning null - dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + Object fileUserStore = ConfigParser.getParsedConfigs().get(UserCoreConstants.FILE_BASED_USER_STORE_AS_PRIMARY); + if (fileUserStore != null && Boolean.parseBoolean(fileUserStore.toString())) { + fileBasedUserStoreMode = true; + hybridRoleManager = new FileBasedHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); } - if (dataSource == null) { - throw new UserStoreException("Data Source is null"); + if (!fileBasedUserStoreMode) { + dataSource = (DataSource) properties.get(UserCoreConstants.DATA_SOURCE); + if (dataSource == null) { + // avoid returning null + dataSource = DatabaseUtil.getRealmDataSource(realmConfig); + } + if (dataSource == null) { + throw new UserStoreException("Data Source is null"); + } + properties.put(UserCoreConstants.DATA_SOURCE, dataSource); + + // hybrid role manager used if only users needs to be read-written. + hybridRoleManager = new JdbcHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); + } else { + hybridRoleManager = new FileBasedHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); } - properties.put(UserCoreConstants.DATA_SOURCE, dataSource); ReadWriteLDAPUserStoreManager.isFirstStartup = (Boolean) properties .get(UserCoreConstants.FIRST_STARTUP_CHECK); - // hybrid role manager used if only users needs to be read-written. - hybridRoleManager = new JdbcHybridRoleManager(dataSource, tenantId, realmConfig, userRealm); // obtain the ldap connection source that was created in // DefaultRealmService. @@ -169,9 +181,11 @@ public ReadWriteLDAPUserStoreManager(RealmConfiguration realmConfig, JNDIUtil.closeContext(dirContext); } this.userRealm = realm; - //persist domain - this.persistDomain(); - doInitialSetup(); + if (!fileBasedUserStoreMode) { + //persist domain + this.persistDomain(); + } + doInitialSetup(fileBasedUserStoreMode); if (realmConfig.isPrimary()) { addInitialAdminData(Boolean.parseBoolean(realmConfig.getAddAdmin()), !isInitSetupDone()); diff --git a/distribution/src/resources/config-tool/templates/conf/internal-apis.xml.j2 b/distribution/src/resources/config-tool/templates/conf/internal-apis.xml.j2 index b0428ebca0..5d1112abfb 100644 --- a/distribution/src/resources/config-tool/templates/conf/internal-apis.xml.j2 +++ b/distribution/src/resources/config-tool/templates/conf/internal-apis.xml.j2 @@ -18,7 +18,7 @@ - + {% if prometheus_api.basic_security_handler.enable == true From efb7d7e0c0bdcf7b5bc5b44c5a8a20babee3d408 Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 1 Nov 2023 15:08:31 +0530 Subject: [PATCH 36/47] Disable existing transaction counter --- .../protocol/jms/JMSInjectHandler.java | 3 -- .../endpoint/protocol/jms/JMSUtils.java | 4 -- .../apis/ManagementInternalApi.java | 6 ++- .../initializer/ServiceBusInitializer.java | 42 +++++++++---------- .../dbscripts/db2/db2_transaction_count.sql | 1 + .../mssql/mssql_transaction_count.sql | 1 + .../mysql/mysql_transaction_count.sql | 1 + .../oracle_rac_transaction_count.sql | 1 + .../oracle/oracle_transaction_count.sql | 1 + .../postgres/postgresql_transaction_count.sql | 1 + .../config-tool/deployment-full.toml | 7 ---- .../mediation-tests/tests-platform/pom.xml | 7 +++- 12 files changed, 35 insertions(+), 40 deletions(-) diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSInjectHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSInjectHandler.java index 4498b19afe..b8b3e0519d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSInjectHandler.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSInjectHandler.java @@ -152,9 +152,6 @@ public boolean invoke(Object object, String name) throws SynapseException { } axis2MsgCtx.setProperty(MessageContext.TRANSPORT_HEADERS, transportHeaders); - // set transaction property - axis2MsgCtx.setProperty(BaseConstants.INTERNAL_TRANSACTION_COUNTED, - msg.getBooleanProperty(BaseConstants.INTERNAL_TRANSACTION_COUNTED)); // set the JMS Message ID as the Message ID of the MessageContext try { if (msg.getJMSMessageID() != null) { diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSUtils.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSUtils.java index 876a00dcc9..aa1cc21a8b 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSUtils.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/jms/JMSUtils.java @@ -429,10 +429,6 @@ public static Map getTransportHeaders(Message message, MessageCo log.error("Error while reading the Transport Headers from JMS Message", e); } - // remove "INTERNAL_TRANSACTION_COUNTED" header from the transport level headers map. - // this property will be maintained in the message context. Therefore, no need to set this in the transport - // headers. - map.remove(BaseConstants.INTERNAL_TRANSACTION_COUNTED); return map; } diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java index b27f7092ee..d3f42cd3fd 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java @@ -101,8 +101,10 @@ public ManagementInternalApi() { resourcesList.add(new ApiResourceAdapter(PREFIX_LOGOUT, new LogoutResource())); resourcesList.add(new ApiResourceAdapter(PREFIX_SERVER_DATA, new MetaDataResource())); resourcesList.add(new LogFilesResource(PREFIX_LOG_FILES)); - resourcesList.add(new ApiResourceAdapter(PREFIX_TRANSACTION + PATH_PARAM_TRANSACTION, - new RequestCountResource())); + + // Disabling this API as this version of transaction counter is deprecated. + // resourcesList.add(new ApiResourceAdapter(PREFIX_TRANSACTION + PATH_PARAM_TRANSACTION, new RequestCountResource())); + resourcesList.add(new ExternalVaultResource(PREFIX_EXTERNAL_VAULTS + PATH_PARAM_EXTERNAL_VAULT_NAME)); resourcesList.add(new ApiResourceAdapter(PREFIX_DATA_SOURCES, new DataSourceResource())); diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java index 6209bb3631..374d65d4c1 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java @@ -58,8 +58,6 @@ import org.wso2.micro.integrator.initializer.deployment.application.deployer.CappDeployer; import org.wso2.micro.integrator.initializer.handler.ProxyLogHandler; import org.wso2.micro.integrator.initializer.handler.SynapseExternalPropertyConfigurator; -import org.wso2.micro.integrator.initializer.handler.transaction.TransactionCountHandler; -import org.wso2.micro.integrator.initializer.handler.transaction.TransactionCountHandlerComponent; import org.wso2.micro.integrator.initializer.persistence.MediationPersistenceManager; import org.wso2.micro.integrator.initializer.services.SynapseConfigurationService; import org.wso2.micro.integrator.initializer.services.SynapseConfigurationServiceImpl; @@ -80,10 +78,7 @@ import java.nio.file.Paths; import java.util.Iterator; import java.util.List; -import java.util.Objects; import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -111,9 +106,9 @@ public class ServiceBusInitializer { private DataSourceService dataSourceService; - private TransactionCountHandlerComponent transactionCountHandlerComponent; - - private ExecutorService transactionCountExecutor; + // This implementation of transaction counter is deprecated + // private TransactionCountHandlerComponent transactionCountHandlerComponent; + // private ExecutorService transactionCountExecutor; @Activate protected void activate(ComponentContext ctxt) { @@ -198,14 +193,14 @@ protected void activate(ComponentContext ctxt) { synapseEnvironment.registerSynapseHandler(new SynapseExternalPropertyConfigurator()); synapseEnvironment.registerSynapseHandler(new ProxyLogHandler()); - // Register internal transaction synapse handler - boolean transactionPropertyEnabled = TransactionCountHandlerComponent.isTransactionPropertyEnabled(); - if (transactionPropertyEnabled) { - transactionCountHandlerComponent = new TransactionCountHandlerComponent(); - transactionCountHandlerComponent.start(dataSourceService); - transactionCountExecutor = Executors.newFixedThreadPool(100); - synapseEnvironment.registerSynapseHandler(new TransactionCountHandler(transactionCountExecutor)); - } + // This implementation of transaction counter is deprecated + // boolean transactionPropertyEnabled = TransactionCountHandlerComponent.isTransactionPropertyEnabled(); + // if (transactionPropertyEnabled) { + // transactionCountHandlerComponent = new TransactionCountHandlerComponent(); + // transactionCountHandlerComponent.start(dataSourceService); + // transactionCountExecutor = Executors.newFixedThreadPool(100); + // synapseEnvironment.registerSynapseHandler(new TransactionCountHandler(transactionCountExecutor)); + // } if (log.isDebugEnabled()) { log.debug("SynapseEnvironmentService Registered"); } @@ -260,12 +255,15 @@ private String getMainSequenceName(String cappName) { @Deactivate protected void deactivate(ComponentContext ctxt) { - if (Objects.nonNull(transactionCountHandlerComponent)) { - transactionCountHandlerComponent.cleanup(); - } - if (Objects.nonNull(transactionCountExecutor)) { - transactionCountExecutor.shutdownNow(); - } + + // This implementation of transaction counter is deprecated + // if (Objects.nonNull(transactionCountHandlerComponent)) { + // transactionCountHandlerComponent.cleanup(); + // } + // if (Objects.nonNull(transactionCountExecutor)) { + // transactionCountExecutor.shutdownNow(); + // } + List handlers = serverManager.getServerContextInformation().getSynapseEnvironment().getSynapseHandlers(); Iterator iterator = handlers.iterator(); while (iterator.hasNext()) { diff --git a/distribution/src/dbscripts/db2/db2_transaction_count.sql b/distribution/src/dbscripts/db2/db2_transaction_count.sql index 903a8e2ebf..f0d763ef44 100644 --- a/distribution/src/dbscripts/db2/db2_transaction_count.sql +++ b/distribution/src/dbscripts/db2/db2_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature CREATE TABLE IF NOT EXISTS CURRENT_STATS ( TIME_STAMP DATE NOT NULL, NODE_ID VARCHAR(40) NOT NULL, diff --git a/distribution/src/dbscripts/mssql/mssql_transaction_count.sql b/distribution/src/dbscripts/mssql/mssql_transaction_count.sql index 28ea169aa0..d244c4057f 100644 --- a/distribution/src/dbscripts/mssql/mssql_transaction_count.sql +++ b/distribution/src/dbscripts/mssql/mssql_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[CURRENT_STATS]') AND TYPE IN (N'U')) CREATE TABLE CURRENT_STATS ( TIME_STAMP DATE NOT NULL, diff --git a/distribution/src/dbscripts/mysql/mysql_transaction_count.sql b/distribution/src/dbscripts/mysql/mysql_transaction_count.sql index aa84912dc5..8662490e90 100644 --- a/distribution/src/dbscripts/mysql/mysql_transaction_count.sql +++ b/distribution/src/dbscripts/mysql/mysql_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature CREATE TABLE IF NOT EXISTS CURRENT_STATS ( TIME_STAMP DATE NOT NULL, NODE_ID VARCHAR(40) NOT NULL, diff --git a/distribution/src/dbscripts/oracle-rac/oracle_rac_transaction_count.sql b/distribution/src/dbscripts/oracle-rac/oracle_rac_transaction_count.sql index aacef1ef4d..cfa8dbdb50 100644 --- a/distribution/src/dbscripts/oracle-rac/oracle_rac_transaction_count.sql +++ b/distribution/src/dbscripts/oracle-rac/oracle_rac_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature CREATE TABLE CURRENT_STATS ( TIME_STAMP DATE NOT NULL, NODE_ID VARCHAR(40) NOT NULL, diff --git a/distribution/src/dbscripts/oracle/oracle_transaction_count.sql b/distribution/src/dbscripts/oracle/oracle_transaction_count.sql index aacef1ef4d..cfa8dbdb50 100644 --- a/distribution/src/dbscripts/oracle/oracle_transaction_count.sql +++ b/distribution/src/dbscripts/oracle/oracle_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature CREATE TABLE CURRENT_STATS ( TIME_STAMP DATE NOT NULL, NODE_ID VARCHAR(40) NOT NULL, diff --git a/distribution/src/dbscripts/postgres/postgresql_transaction_count.sql b/distribution/src/dbscripts/postgres/postgresql_transaction_count.sql index 903a8e2ebf..f0d763ef44 100644 --- a/distribution/src/dbscripts/postgres/postgresql_transaction_count.sql +++ b/distribution/src/dbscripts/postgres/postgresql_transaction_count.sql @@ -1,3 +1,4 @@ +-- Related to the deprecated transaction counter feature CREATE TABLE IF NOT EXISTS CURRENT_STATS ( TIME_STAMP DATE NOT NULL, NODE_ID VARCHAR(40) NOT NULL, diff --git a/distribution/src/resources/config-tool/deployment-full.toml b/distribution/src/resources/config-tool/deployment-full.toml index 655fda4f26..7ecc1a91f5 100644 --- a/distribution/src/resources/config-tool/deployment-full.toml +++ b/distribution/src/resources/config-tool/deployment-full.toml @@ -1098,13 +1098,6 @@ user.password = "pwd-2" [internal_apis.file_user_store] enable = false -#################### Transaction Counter Handler ############################ -[transaction_counter] -enable = true -data_source = "WSO2_COORDINATION_DB" -update_interval = 2 - - #################### Service Catalog ####################################### [[service_catalog]] apim_host = "https://127.0.0.1:9443" diff --git a/integration/mediation-tests/tests-platform/pom.xml b/integration/mediation-tests/tests-platform/pom.xml index e30198d2f3..cdef03580d 100644 --- a/integration/mediation-tests/tests-platform/pom.xml +++ b/integration/mediation-tests/tests-platform/pom.xml @@ -35,8 +35,11 @@ tests-rabbitmq tests-coordination tests-userstore - tests-transaction - + + + + + From 2849ea36a8c9bcf42f8c2382735320d66069b356 Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 1 Nov 2023 15:29:59 +0530 Subject: [PATCH 37/47] Remove transaction counter GitHub workflow --- .github/workflows/transaction-tests.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/transaction-tests.yml diff --git a/.github/workflows/transaction-tests.yml b/.github/workflows/transaction-tests.yml deleted file mode 100644 index 82c330bee5..0000000000 --- a/.github/workflows/transaction-tests.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Transaction Counter tests (OpenJDK11 & MySQL) - -on: [push, pull_request] - -jobs: - ubuntu: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Set up MySQL - run: | - sudo /etc/init.d/mysql start - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - distribution: 'adopt' - java-version: 11 - - name: Build in JDK 11 - run: mvn -B clean install -DskipTests --file pom.xml - - name: Transaction Counter Tests - run: mvn -B clean install --file integration/pom.xml -P transaction-tests From 38df0715bbf1eba40865c39b03415b5be6c7dc95 Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 1 Nov 2023 15:49:59 +0530 Subject: [PATCH 38/47] Deprecate transaction counter --- .../management/apis/RequestCountResource.java | 3 +++ .../initializer/ServiceBusInitializer.java | 21 ------------------- .../initializer/handler/DataHolder.java | 4 +++- .../transaction/TransactionConstants.java | 4 ++++ .../transaction/TransactionCountHandler.java | 4 ++++ .../TransactionCountHandlerComponent.java | 3 ++- .../TransactionCounterException.java | 4 ++++ ...sactionCounterInitializationException.java | 4 ++++ .../transaction/security/CryptoUtil.java | 3 +++ .../transaction/store/TransactionStore.java | 3 +++ 10 files changed, 30 insertions(+), 23 deletions(-) diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RequestCountResource.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RequestCountResource.java index ed8c5471cc..a104960792 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RequestCountResource.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RequestCountResource.java @@ -50,6 +50,9 @@ /** + * This feature is deprecated and will be removed in a future release. + * @deprecated + * * Resource for a retrieving aggregated transaction count. *

* Handles resources in the form "management/transactions/{param}" diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java index 374d65d4c1..644de0b5f2 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/ServiceBusInitializer.java @@ -106,10 +106,6 @@ public class ServiceBusInitializer { private DataSourceService dataSourceService; - // This implementation of transaction counter is deprecated - // private TransactionCountHandlerComponent transactionCountHandlerComponent; - // private ExecutorService transactionCountExecutor; - @Activate protected void activate(ComponentContext ctxt) { @@ -193,14 +189,6 @@ protected void activate(ComponentContext ctxt) { synapseEnvironment.registerSynapseHandler(new SynapseExternalPropertyConfigurator()); synapseEnvironment.registerSynapseHandler(new ProxyLogHandler()); - // This implementation of transaction counter is deprecated - // boolean transactionPropertyEnabled = TransactionCountHandlerComponent.isTransactionPropertyEnabled(); - // if (transactionPropertyEnabled) { - // transactionCountHandlerComponent = new TransactionCountHandlerComponent(); - // transactionCountHandlerComponent.start(dataSourceService); - // transactionCountExecutor = Executors.newFixedThreadPool(100); - // synapseEnvironment.registerSynapseHandler(new TransactionCountHandler(transactionCountExecutor)); - // } if (log.isDebugEnabled()) { log.debug("SynapseEnvironmentService Registered"); } @@ -255,15 +243,6 @@ private String getMainSequenceName(String cappName) { @Deactivate protected void deactivate(ComponentContext ctxt) { - - // This implementation of transaction counter is deprecated - // if (Objects.nonNull(transactionCountHandlerComponent)) { - // transactionCountHandlerComponent.cleanup(); - // } - // if (Objects.nonNull(transactionCountExecutor)) { - // transactionCountExecutor.shutdownNow(); - // } - List handlers = serverManager.getServerContextInformation().getSynapseEnvironment().getSynapseHandlers(); Iterator iterator = handlers.iterator(); while (iterator.hasNext()) { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/DataHolder.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/DataHolder.java index 0aae95fef9..8b5180977c 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/DataHolder.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/DataHolder.java @@ -21,7 +21,9 @@ import org.wso2.micro.integrator.initializer.handler.transaction.store.TransactionStore; /** - * DataHolder class for transaction count component. + * This feature is deprecated and will be removed in a future release. + * + * @deprecated */ public class DataHolder { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionConstants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionConstants.java index 55f6b20a41..34171f9679 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionConstants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionConstants.java @@ -22,6 +22,10 @@ import java.io.File; +/** + * This feature is deprecated and will be removed in a future release. + * @deprecated + */ public class TransactionConstants { public static final String PUBLIC_KEY = System.getProperty(ServerConstants.CARBON_HOME) + File.separator diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandler.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandler.java index 0612458fda..14ec63862f 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandler.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandler.java @@ -30,6 +30,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +/** + * This feature is deprecated and will be removed in a future release. + * @deprecated + */ public class TransactionCountHandler extends AbstractSynapseHandler { private static final Log LOG = LogFactory.getLog(TransactionCountHandler.class); diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandlerComponent.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandlerComponent.java index f92def6cc8..977e99c5e4 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandlerComponent.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/TransactionCountHandlerComponent.java @@ -39,7 +39,8 @@ import javax.sql.DataSource; /** - * This class represents the internal transaction count handler component. + * This feature is deprecated and will be removed in a future release. + * @deprecated */ public class TransactionCountHandlerComponent { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterException.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterException.java index 04b417ec13..349e9bd553 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterException.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterException.java @@ -18,6 +18,10 @@ package org.wso2.micro.integrator.initializer.handler.transaction.exception; +/** + * This feature is deprecated and will be removed in a future release. + * @deprecated + */ public class TransactionCounterException extends Exception { public TransactionCounterException(String msg, Exception e) { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterInitializationException.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterInitializationException.java index b1ffc5be03..8617c685b4 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterInitializationException.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/exception/TransactionCounterInitializationException.java @@ -18,6 +18,10 @@ package org.wso2.micro.integrator.initializer.handler.transaction.exception; +/** + * This feature is deprecated and will be removed in a future release. + * @deprecated + */ public class TransactionCounterInitializationException extends Exception { public TransactionCounterInitializationException(String msg) { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/security/CryptoUtil.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/security/CryptoUtil.java index 499ea311e4..ffa5da6b69 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/security/CryptoUtil.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/security/CryptoUtil.java @@ -42,6 +42,9 @@ import javax.xml.bind.DatatypeConverter; /** + * This feature is deprecated and will be removed in a future release. + * @deprecated + * * This class will provide the required methods to encrypt a given value. */ public class CryptoUtil { diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/store/TransactionStore.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/store/TransactionStore.java index ea561552f5..0720965495 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/store/TransactionStore.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/handler/transaction/store/TransactionStore.java @@ -28,6 +28,9 @@ import javax.sql.DataSource; /** + * This feature is deprecated and will be removed in a future release. + * @deprecated + * * The layer which connects to the transaction data. */ public class TransactionStore { From 645d42c97ba2327d8e731b84722efd198ad24799 Mon Sep 17 00:00:00 2001 From: Isuru Wijesiri Date: Wed, 1 Nov 2023 23:01:06 +0530 Subject: [PATCH 39/47] Disable management API for transaction counter --- .../integrator/management/apis/ManagementInternalApi.java | 3 --- integration/mediation-tests/tests-platform/pom.xml | 3 --- 2 files changed, 6 deletions(-) diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java index d3f42cd3fd..78a6ee2a5c 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/ManagementInternalApi.java @@ -102,9 +102,6 @@ public ManagementInternalApi() { resourcesList.add(new ApiResourceAdapter(PREFIX_SERVER_DATA, new MetaDataResource())); resourcesList.add(new LogFilesResource(PREFIX_LOG_FILES)); - // Disabling this API as this version of transaction counter is deprecated. - // resourcesList.add(new ApiResourceAdapter(PREFIX_TRANSACTION + PATH_PARAM_TRANSACTION, new RequestCountResource())); - resourcesList.add(new ExternalVaultResource(PREFIX_EXTERNAL_VAULTS + PATH_PARAM_EXTERNAL_VAULT_NAME)); resourcesList.add(new ApiResourceAdapter(PREFIX_DATA_SOURCES, new DataSourceResource())); diff --git a/integration/mediation-tests/tests-platform/pom.xml b/integration/mediation-tests/tests-platform/pom.xml index cdef03580d..896ac75ce3 100644 --- a/integration/mediation-tests/tests-platform/pom.xml +++ b/integration/mediation-tests/tests-platform/pom.xml @@ -36,9 +36,6 @@ tests-coordination tests-userstore - - - From 63c069b03d8bd61bac00c1ae7925a211b198b829 Mon Sep 17 00:00:00 2001 From: chanikag Date: Mon, 6 Nov 2023 16:30:21 +0530 Subject: [PATCH 40/47] Fix vfs testcases connect to RSA supported sftp server The tescases are failing after upgrading to vfs version to com.github.mwiede 0.2.4 The RSA/SHA1 signature algorithm is disabled by default. For legacy sftp severs jsch.server_host_key + jsch.client_pubkey properties has to set with the RSA algorithm --- .../esb/vfs/transport/test/ESBJAVA3470.java | 16 + ...SPasswordSecurityWithLargekeyTestCase.java | 15 +- .../artifacts/ESB/vfs/micro-integrator.bat | 212 +++++++++++ .../artifacts/ESB/vfs/micro-integrator.sh | 351 ++++++++++++++++++ 4 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.bat create mode 100755 integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.sh diff --git a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA3470.java b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA3470.java index f0ebcba762..1f97b563fa 100644 --- a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA3470.java +++ b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA3470.java @@ -38,6 +38,7 @@ import org.testng.annotations.Test; import org.wso2.carbon.automation.engine.annotations.ExecutionEnvironment; import org.wso2.carbon.automation.engine.annotations.SetEnvironment; +import org.wso2.carbon.base.CarbonBaseUtils; import org.wso2.carbon.proxyadmin.stub.ProxyServiceAdminProxyAdminException; import org.wso2.carbon.utils.ServerConstants; import org.wso2.esb.integration.common.utils.ESBIntegrationTest; @@ -79,6 +80,8 @@ public class ESBJAVA3470 extends ESBIntegrationTest { private File SFTPFolder; private String carbonHome; private ServerConfigurationManager serverConfigurationManager; + private static final String SH_FILE_NAME = "micro-integrator.sh"; + private static final String BAT_FILE_NAME = "micro-integrator.bat"; public static KeyPairProvider createTestHostKeyProvider(Path path) { SimpleGeneratorHostKeyProvider keyProvider = new SimpleGeneratorHostKeyProvider(); @@ -97,6 +100,18 @@ public void deployService() throws Exception { setupSftpFolders(carbonHome); setupSftpServer(carbonHome); Thread.sleep(15000); + File newShFile = new File( + getESBResourceLocation() + File.separator + "vfs" + File.separator + SH_FILE_NAME); + File oldShFile = + new File(CarbonBaseUtils.getCarbonHome() + File.separator + "bin" + File.separator + SH_FILE_NAME); + serverConfigurationManager.applyConfigurationWithoutRestart(newShFile, oldShFile, true); + File newBatFile = new File( + getESBResourceLocation() + File.separator + "vfs" + File.separator + BAT_FILE_NAME); + File oldBatFile = + new File(CarbonBaseUtils.getCarbonHome() + File.separator + "bin" + File.separator + BAT_FILE_NAME); + serverConfigurationManager.applyConfigurationWithoutRestart(newBatFile, oldBatFile, true); + serverConfigurationManager.restartGracefully(); + super.init(); } @Test(groups = "wso2.esb", description = "VFS absolute path test for sftp") @@ -154,6 +169,7 @@ public void stopSFTPServer() throws Exception { //sshd.stop(); log.info("SFTP Server stopped successfully"); super.cleanup(); + serverConfigurationManager.restoreToLastMIConfiguration(); } /** diff --git a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA4770VFSPasswordSecurityWithLargekeyTestCase.java b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA4770VFSPasswordSecurityWithLargekeyTestCase.java index 6fcd4c0b86..2ef805a018 100644 --- a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA4770VFSPasswordSecurityWithLargekeyTestCase.java +++ b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/vfs/transport/test/ESBJAVA4770VFSPasswordSecurityWithLargekeyTestCase.java @@ -66,6 +66,8 @@ public class ESBJAVA4770VFSPasswordSecurityWithLargekeyTestCase extends ESBInteg private String outputFolderName = "out"; private static final String USERNAME = "SFTPUser"; private static final String PASSWORD = "SFTP321"; + private static final String SH_FILE_NAME = "micro-integrator.sh"; + private static final String BAT_FILE_NAME = "micro-integrator.bat"; @BeforeClass(alwaysRun = true) public void runFTPServer() throws Exception { @@ -116,9 +118,20 @@ public void runFTPServer() throws Exception { // replace the axis2.xml enabled vfs transfer and restart the ESB server gracefully. serverConfigurationManager = new ServerConfigurationManager(context); - serverConfigurationManager.applyMIConfiguration(new File( + serverConfigurationManager.applyMIConfigurationWithRestart(new File( getClass().getResource("/artifacts/ESB/synapseconfig/" + "vfsTransport/ESBJAVA4770/deployment.toml") .getPath())); + File newShFile = new File( + getESBResourceLocation() + File.separator + "vfs" + File.separator + SH_FILE_NAME); + File oldShFile = + new File(CarbonBaseUtils.getCarbonHome() + File.separator + "bin" + File.separator + SH_FILE_NAME); + serverConfigurationManager.applyConfigurationWithoutRestart(newShFile, oldShFile, true); + File newBatFile = new File( + getESBResourceLocation() + File.separator + "vfs" + File.separator + BAT_FILE_NAME); + File oldBatFile = + new File(CarbonBaseUtils.getCarbonHome() + File.separator + "bin" + File.separator + BAT_FILE_NAME); + serverConfigurationManager.applyConfigurationWithoutRestart(newBatFile, oldBatFile, true); + serverConfigurationManager.restartGracefully(); super.init(); } diff --git a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.bat b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.bat new file mode 100644 index 0000000000..749d336833 --- /dev/null +++ b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.bat @@ -0,0 +1,212 @@ +@echo off +REM --------------------------------------------------------------------------- +REM Copyright 2018 WSO2, Inc. http://www.wso2.org +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +rem --------------------------------------------------------------------------- +rem Main Script for WSO2 Carbon +rem +rem Environment Variable Prequisites +rem +rem CARBON_HOME Home of CARBON installation. If not set I will try +rem to figure it out. +rem +rem JAVA_HOME Must point at your Java Development Kit installation. +rem +rem JAVA_OPTS (Optional) Java runtime options used when the commands +rem is executed. +rem --------------------------------------------------------------------------- + +rem --------- NOTE: This is an edited wso2server.sh script to facilitate +rem spark environment variables for WSO2DAS! +rem ----- if JAVA_HOME is not set we're not happy ------------------------------ +:checkJava + +if "%JAVA_HOME%" == "" goto noJavaHome +if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome +goto checkServer + +:noJavaHome +echo "You must set the JAVA_HOME variable before running CARBON." +goto end + +rem ----- Only set CARBON_HOME if not already set ---------------------------- +:checkServer +rem %~sdp0 is expanded pathname of the current script under NT with spaces in the path removed +SET CARBON_HOME=%~sdp0.. +SET curDrive=%cd:~0,1% +SET wsasDrive=%CARBON_HOME:~0,1% +if not "%curDrive%" == "%wsasDrive%" %wsasDrive%: + +rem find CARBON_HOME if it does not exist due to either an invalid value passed +rem by the user or the %0 problem on Windows 9x +if not exist "%CARBON_HOME%\bin\version.txt" goto noServerHome + +set AXIS2_HOME=%CARBON_HOME% +goto updateClasspath + +:noServerHome +echo CARBON_HOME is set incorrectly or CARBON could not be located. Please set CARBON_HOME. +goto end + +rem ----- update classpath ----------------------------------------------------- +:updateClasspath + +setlocal EnableDelayedExpansion +cd %CARBON_HOME% +set CARBON_CLASSPATH= +FOR %%C in ("%CARBON_HOME%\bin\*.jar") DO set CARBON_CLASSPATH=!CARBON_CLASSPATH!;".\bin\%%~nC%%~xC" + +set CARBON_CLASSPATH="%JAVA_HOME%\lib\tools.jar";%CARBON_CLASSPATH%; + +FOR %%D in ("%CARBON_HOME%\wso2\lib\*.jar") DO set CARBON_CLASSPATH=!CARBON_CLASSPATH!;".\wso2\lib\%%~nD%%~xD" + +rem ----- Process the input command ------------------------------------------- + +rem Slurp the command line arguments. This loop allows for an unlimited number +rem of arguments (up to the command line limit, anyway). + + +:setupArgs +if ""%1""=="""" goto doneStart + +if ""%1""==""-run"" goto commandLifecycle +if ""%1""==""--run"" goto commandLifecycle +if ""%1""==""run"" goto commandLifecycle + +if ""%1""==""-restart"" goto commandLifecycle +if ""%1""==""--restart"" goto commandLifecycle +if ""%1""==""restart"" goto commandLifecycle + +if ""%1""==""debug"" goto commandDebug +if ""%1""==""-debug"" goto commandDebug +if ""%1""==""--debug"" goto commandDebug + +if ""%1""==""version"" goto commandVersion +if ""%1""==""-version"" goto commandVersion +if ""%1""==""--version"" goto commandVersion + +if ""%1""==""stop"" goto stopServer +if ""%1""==""-stop"" goto stopServer +if ""%1""==""--stop"" goto stopServer + +if ""%1""==""car"" goto setCar +if ""%1""==""-car"" goto setCar +if ""%1""==""--car"" goto setCar + +shift +goto setupArgs + +rem ----- commandVersion ------------------------------------------------------- +:commandVersion +shift +type "%CARBON_HOME%\bin\version.txt" +type "%CARBON_HOME%\bin\wso2carbon-version.txt" +goto end + +rem ----- commandDebug --------------------------------------------------------- +:commandDebug +shift +set DEBUG_PORT=%1 +if "%DEBUG_PORT%"=="" goto noDebugPort +if not "%JAVA_OPTS%"=="" echo Warning !!!. User specified JAVA_OPTS will be ignored, once you give the --debug option. +set JAVA_OPTS=-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=%DEBUG_PORT% +echo Please start the remote debugging client to continue... +goto findJdk + +:noDebugPort +echo Please specify the debug port after the --debug option +goto end + +:stopServer +set /p processId= < %CARBON_HOME%\wso2carbon.pid +echo Stopping the Micro Integrator Server +taskkill /F /PID %processId% +goto end + +rem ----- commandLifecycle ----------------------------------------------------- +:commandLifecycle +goto findJdk + +:doneStart +if "%OS%"=="Windows_NT" @setlocal +if "%OS%"=="WINNT" @setlocal + +:setCar +set CAR_NAME=%2 + +rem ---------- Handle the SSL Issue with proper JDK version -------------------- +rem find the version of the jdk +:findJdk + +set CMD=RUN %* + +:checkJdk +PATH %PATH%;%JAVA_HOME%\bin\ +for /f tokens^=2-5^ delims^=.-_+^" %%j in ('java -fullversion 2^>^&1') do set "JAVA_VERSION=%%j%%k" +if %JAVA_VERSION% LSS 110 goto unknownJdk +goto supportedJdk + +:unknownJdk +echo Starting WSO2 MI (in unsupported JDK %JAVA_VERSION%) +echo [ERROR] WSO2 MI is supported only between JDK 11 and JDK 17" +goto end + +:supportedJdk +goto runServer + +rem ----------------- Execute The Requested Command ---------------------------- + +:runServer +cd %CARBON_HOME% + +rem ------------------ Remove tmp folder on startup ----------------------------- +set TMP_DIR=%CARBON_HOME%\tmp +cd "%TMP_DIR%" +del *.* /s /q > nul +FOR /d %%G in ("*.*") DO rmdir %%G /s /q +cd .. + +rem ---------- Add jars to classpath --c _CLASSPATH% + +if %JAVA_VERSION% GEQ 110 set CARBON_CLASSPATH=.\wso2\lib\*;%CARBON_CLASSPATH% +if %JAVA_VERSION% GEQ 110 set JAVA_VER_BASED_OPTS=--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.util=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED + +rem ---------------- Setting default profile for Runtime if not parsed -------------- + +set profileSet=false + +for %%x in (%*) do ( + if "%%x" == "%x:-Dprofile%" ( + set profileSet=true + ) +) + +if "%profileSet%" == "false" ( + set profile=-Dprofile=micro-integrator-default +) + +set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof" -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% -Djava.endorsed.dirs=%JAVA_ENDORSED% -DandesConfig=broker.xml -Dcarbon.registry.root=/ -Dcarbon.home="%CARBON_HOME%" -Dlogfiles.home="%CARBON_HOME%\repository\logs" -Dwso2.server.standalone=true -Djava.command="%JAVA_HOME%\bin\java" -Djava.opts="%JAVA_OPTS%" -Djava.io.tmpdir="%CARBON_HOME%\tmp" -Dcatalina.base="%CARBON_HOME%\wso2\lib\tomcat" -Dwso2.carbon.xml="%CARBON_HOME%\conf\carbon.xml" -Dwso2.registry.xml="%CARBON_HOME%\conf\registry.xml" -Dwso2.user.mgt.xml="%CARBON_HOME%\conf\user-mgt.xml" -Dwso2.transports.xml="%CARBON_HOME%\conf\mgt-transports.xml" -Djava.util.logging.config.file="%CARBON_HOME%\conf\log4j.properties" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -DNonUserCoreMode=true -DNonRegistryMode=true -Dcarbon.logs.path="%CARBON_HOME%\repository\logs" -Dcomponents.repo="%CARBON_HOME%\wso2\components\plugins" -Dcarbon.config.dir.path="%CARBON_HOME%\conf" -Dcarbon.components.dir.path="%CARBON_HOME%\wso2\components" -Dcarbon.dropins.dir.path="%CARBON_HOME%\dropins" -Dcarbon.external.lib.dir.path="%CARBON_HOME%\lib" -Dcarbon.patches.dir.path="%CARBON_HOME%\patches" -Dcarbon.internal.lib.dir.path="%CARBON_HOME%\wso2\lib" -Dconf.location="%CARBON_HOME%\conf" -Dcom.atomikos.icatch.file="%CARBON_HOME%\wso2\lib\transactions.properties" -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler -Dcom.atomikos.icatch.hide_init_file_path="true" -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dcom.sun.jndi.ldap.connect.pool.authentication=simple -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 -Dorg.terracotta.quartz.skipUpdateCheck=true -Dcarbon.classpath=%CARBON_CLASSPATH% -Dfile.encoding=UTF8 -Dlogger.server.name="micro-integrator" -Dqpid.conf="\conf\advanced" -Dproperties.file.path=default -Djavax.xml.xpath.XPathFactory:http://java.sun.com/jaxp/xpath/dom=net.sf.saxon.xpath.XPathFactoryImpl -DavoidConfigHashRead=true -DenableReadinessProbe=true -DenableLivenessProbe=true -DenableManagementApi=true -DskipStartupExtensions=false -Dautomation.mode.seq.car.name="%CAR_NAME%" -Dlog4j2.contextSelector="org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" -Dorg.ops4j.pax.logging.logReaderEnabled=false -Djsch.server_host_key=ssh-rsa,ssh-dss -Djsch.client_pubkey=ssh-rsa,ssh-dss -Dorg.ops4j.pax.logging.eventAdminEnabled=false %JAVA_VER_BASED_OPTS% %profile% -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" + +:runJava +rem echo JAVA_HOME environment variable is set to %JAVA_HOME% +rem echo CARBON_HOME environment variable is set to %CARBON_HOME% +"%JAVA_HOME%\bin\java" %CMD_LINE_ARGS% org.wso2.micro.integrator.bootstrap.Bootstrap %CMD% +if "%ERRORLEVEL%"=="121" goto runJava +:end +goto endlocal + +:endlocal + +:END diff --git a/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.sh b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.sh new file mode 100755 index 0000000000..47636c2e8b --- /dev/null +++ b/integration/mediation-tests/tests-patches/src/test/resources/artifacts/ESB/vfs/micro-integrator.sh @@ -0,0 +1,351 @@ +#!/bin/sh +# micro-integrator.sh +# ---------------------------------------------------------------------------- +# Copyright 2018 WSO2, Inc. http://www.wso2.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cygwin=false; +darwin=false; +os400=false; +mingw=false; +case "`uname`" in +CYGWIN*) cygwin=true;; +MINGW*) mingw=true;; +OS400*) os400=true;; +Darwin*) darwin=true + if [ -z "$JAVA_VERSION" ] ; then + JAVA_VERSION="CurrentJDK" + else + echo "Using Java version: $JAVA_VERSION" + fi + if [ -z "$JAVA_HOME" ] ; then + JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home + fi + ;; +esac + +# resolve links - $0 may be a softlink +PRG="$0" + +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done + +# Get standard environment variables +PRGDIR=`dirname "$PRG"` + +# Only set CARBON_HOME if not already set +[ -z "$CARBON_HOME" ] && CARBON_HOME=`cd "$PRGDIR/.." ; pwd` + +# Set AXIS2_HOME. Needed for One Click JAR Download +AXIS2_HOME="$CARBON_HOME" + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CARBON_HOME" ] && CARBON_HOME=`cygpath --unix "$CARBON_HOME"` + [ -n "$AXIS2_HOME" ] && CARBON_HOME=`cygpath --unix "$CARBON_HOME"` +fi + +# For OS400 +if $os400; then + # Set job priority to standard for interactive (interactive - 6) by using + # the interactive priority - 6, the helper threads that respond to requests + # will be running at the same priority as interactive jobs. + COMMAND='chgjob job('$JOBNAME') runpty(6)' + system $COMMAND + + # Enable multi threading + QIBM_MULTI_THREADED=Y + export QIBM_MULTI_THREADED +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$CARBON_HOME" ] && + CARBON_HOME="`(cd "$CARBON_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + [ -n "$AXIS2_HOME" ] && + CARBON_HOME="`(cd "$CARBON_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD=java + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." + echo " CARBON cannot execute $JAVACMD" + exit 1 +fi + +# if JAVA_HOME is not set we're not happy +if [ -z "$JAVA_HOME" ]; then + echo "You must set the JAVA_HOME variable before running CARBON." + exit 1 +fi + +if [ -e "$CARBON_HOME/wso2carbon.pid" ]; then + PID=`cat "$CARBON_HOME"/wso2carbon.pid` +fi + +# ----- Process the input command ---------------------------------------------- +args="" +for c in $* +do + if [ "$c" = "--debug" ] || [ "$c" = "-debug" ] || [ "$c" = "debug" ]; then + CMD="--debug" + continue + elif [ "$CMD" = "--debug" ]; then + if [ -z "$PORT" ]; then + PORT=$c + fi + elif [ "$c" = "--stop" ] || [ "$c" = "-stop" ] || [ "$c" = "stop" ]; then + CMD="stop" + elif [ "$c" = "--start" ] || [ "$c" = "-start" ] || [ "$c" = "start" ]; then + CMD="start" + elif [ "$c" = "--version" ] || [ "$c" = "-version" ] || [ "$c" = "version" ]; then + CMD="version" + elif [ "$c" = "--restart" ] || [ "$c" = "-restart" ] || [ "$c" = "restart" ]; then + CMD="restart" + elif [ "$c" = "--car" ] || [ "$c" = "-car" ] || [ "$c" = "car" ]; then + ARGUMENT="car" + OPTARG="$2" + else + args="$args $c" + fi +done + +if [ "$ARGUMENT" = "car" ]; then + CAR_NAME="$OPTARG" +fi + +if [ "$CMD" = "--debug" ]; then + if [ "$PORT" = "" ]; then + echo " Please specify the debug port after the --debug option" + exit 1 + fi + if [ -n "$JAVA_OPTS" ]; then + echo "Warning !!!. User specified JAVA_OPTS will be ignored, once you give the --debug option." + fi + CMD="RUN" + JAVA_OPTS="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=$PORT" + echo "Please start the remote debugging client to continue..." +elif [ "$CMD" = "start" ]; then + if [ -e "$CARBON_HOME/wso2carbon.pid" ]; then + if ps -p $PID > /dev/null ; then + echo "Process is already running" + exit 0 + fi + fi + export CARBON_HOME="$CARBON_HOME" +# using nohup sh to avoid erros in solaris OS.TODO + nohup sh "$CARBON_HOME"/bin/micro-integrator.sh $args > /dev/null 2>&1 & + exit 0 +elif [ "$CMD" = "stop" ]; then + export CARBON_HOME="$CARBON_HOME" + kill -term `cat "$CARBON_HOME"/wso2carbon.pid` + exit 0 +elif [ "$CMD" = "restart" ]; then + export CARBON_HOME="$CARBON_HOME" + kill -term `cat "$CARBON_HOME"/wso2carbon.pid` + process_status=0 + pid=`cat "$CARBON_HOME"/wso2carbon.pid` + while [ "$process_status" -eq "0" ] + do + sleep 1; + ps -p$pid 2>&1 > /dev/null + process_status=$? + done + +# using nohup sh to avoid erros in solaris OS.TODO + nohup sh "$CARBON_HOME"/bin/micro-integrator.sh $args > /dev/null 2>&1 & + exit 0 + +elif [ "$CMD" = "version" ]; then + cat "$CARBON_HOME"/bin/version.txt + cat "$CARBON_HOME"/bin/wso2carbon-version.txt + exit 0 +fi + +# ---------- Handle the SSL Issue with proper JDK version -------------------- +java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') +java_version_formatted=$(echo "$java_version" | awk -F. '{printf("%02d%02d",$1,$2);}') +if [ $java_version_formatted -lt 1100 ] || [ $java_version_formatted -gt 1700 ]; then + echo " Starting WSO2 MI (in unsupported JDK)" + echo " [ERROR] WSO2 MI is supported only between JDK 11 and JDK 17" + exit 0 +fi + +CARBON_XBOOTCLASSPATH="" +for f in "$CARBON_HOME"/wso2/lib/xboot/*.jar +do + if [ "$f" != "$CARBON_HOME/wso2/lib/xboot/*.jar" ];then + CARBON_XBOOTCLASSPATH="$CARBON_XBOOTCLASSPATH":$f + fi +done + +CARBON_CLASSPATH="" +if [ -e "$JAVA_HOME/lib/tools.jar" ]; then + CARBON_CLASSPATH="$JAVA_HOME/../lib/tools.jar" +fi +for f in "$CARBON_HOME"/bin/*.jar +do + if [ "$f" != "$CARBON_HOME/bin/*.jar" ];then + CARBON_CLASSPATH="$CARBON_CLASSPATH":$f + fi +done +for t in "$CARBON_HOME"/wso2/lib/*.jar +do + CARBON_CLASSPATH="$CARBON_CLASSPATH":$t +done + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + JAVA_HOME=`cygpath --absolute --windows "$JAVA_HOME"` + CARBON_HOME=`cygpath --absolute --windows "$CARBON_HOME"` + AXIS2_HOME=`cygpath --absolute --windows "$CARBON_HOME"` + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + CARBON_CLASSPATH=`cygpath --path --windows "$CARBON_CLASSPATH"` + CARBON_XBOOTCLASSPATH=`cygpath --path --windows "$CARBON_XBOOTCLASSPATH"` +fi + +# ----- Execute The Requested Command ----------------------------------------- + +# echo JAVA_HOME environment variable is set to $JAVA_HOME +# echo CARBON_HOME environment variable is set to "$CARBON_HOME" + +cd "$CARBON_HOME" + +TMP_DIR="$CARBON_HOME"/tmp +if [ -d "$TMP_DIR" ]; then +rm -rf "$TMP_DIR"/* +fi + +START_EXIT_STATUS=121 +status=$START_EXIT_STATUS + +if [ -z "$JVM_MEM_OPTS" ]; then + java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') + JVM_MEM_OPTS="-Xms256m -Xmx1024m" + if [ "$java_version" \< "1.8" ]; then + JVM_MEM_OPTS="$JVM_MEM_OPTS" + fi +fi +# echo "Using Java memory options: $JVM_MEM_OPTS" + +#setting up profile parameter for runtime in MB +PROFILE_SELECTED="false" +for i in "$@"; do + if echo "$i" | grep -q "Dprofile"; then + PROFILE_SELECTED="true" + fi +done + +if [ "$PROFILE_SELECTED" = false ] ; then + NODE_PARAMS="$NODE_PARAMS -Dprofile=micro-integrator-default" +fi + +#To monitor a Carbon server in remote JMX mode on linux host machines, set the below system property. +# -Djava.rmi.server.hostname="your.IP.goes.here" + +JAVA_VER_BASED_OPTS="" + +if [ $java_version_formatted -ge 1100 ]; then + JAVA_VER_BASED_OPTS="--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.util=ALL-UNNAMED --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED" +fi + +while [ "$status" = "$START_EXIT_STATUS" ] +do + $JAVACMD \ + -Xbootclasspath/a:"$CARBON_XBOOTCLASSPATH" \ + $JVM_MEM_OPTS \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="$CARBON_HOME/repository/logs/heap-dump.hprof" \ + $JAVA_OPTS \ + -Dcom.sun.management.jmxremote \ + -classpath "$CARBON_CLASSPATH" \ + -Djava.io.tmpdir="$CARBON_HOME/tmp" \ + -Dcatalina.base="$CARBON_HOME/wso2/lib/tomcat" \ + -Dwso2.server.standalone=true \ + -Dcarbon.registry.root=/ \ + -Djava.command="$JAVACMD" \ + -Dqpid.conf="/conf/advanced/" \ + $JAVA_VER_BASED_OPTS \ + -Dcarbon.home="$CARBON_HOME" \ + -Dlogfiles.home="$CARBON_HOME/repository/logs" \ + -Dlogger.server.name="micro-integrator" \ + -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \ + -Dcarbon.config.dir.path="$CARBON_HOME/conf" \ + -Dcarbon.repository.dir.path="$CARBON_HOME/repository" \ + -Dcarbon.components.dir.path="$CARBON_HOME/wso2/components" \ + -Dcarbon.dropins.dir.path="$CARBON_HOME/dropins" \ + -Dcarbon.external.lib.dir.path="$CARBON_HOME/lib" \ + -Dcarbon.patches.dir.path="$CARBON_HOME/patches" \ + -Dcarbon.internal.lib.dir.path="$CARBON_HOME/wso2/lib" \ + -Dei.extendedURIBasedDispatcher=org.wso2.micro.integrator.core.handlers.IntegratorStatefulHandler \ + -Djava.util.logging.config.file="$CARBON_HOME/conf/etc/logging-bridge.properties" \ + -Dcomponents.repo="$CARBON_HOME/wso2/components/plugins" \ + -Dconf.location="$CARBON_HOME/conf" \ + -Dcom.atomikos.icatch.file="$CARBON_HOME/wso2/lib/transactions.properties" \ + -Dcom.atomikos.icatch.hide_init_file_path=true \ + -Dorg.apache.jasper.compiler.Parser.STRICT_QUOTE_ESCAPING=false \ + -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true \ + -Dcom.sun.jndi.ldap.connect.pool.authentication=simple \ + -Dcom.sun.jndi.ldap.connect.pool.timeout=3000 \ + -Dorg.terracotta.quartz.skipUpdateCheck=true \ + -Djava.security.egd=file:/dev/./urandom \ + -Dfile.encoding=UTF8 \ + -Djava.net.preferIPv4Stack=true \ + -DNonRegistryMode=true \ + -DNonUserCoreMode=true \ + -Dcom.ibm.cacheLocalHost=true \ + -Dcarbon.use.registry.repo=false \ + -DworkerNode=false \ + -Dorg.apache.cxf.io.CachedOutputStream.Threshold=104857600 \ + -Djavax.xml.xpath.XPathFactory:http://java.sun.com/jaxp/xpath/dom=net.sf.saxon.xpath.XPathFactoryImpl \ + -DavoidConfigHashRead=true \ + -Dproperties.file.path=default \ + -DenableReadinessProbe=true \ + -DenableLivenessProbe=true \ + -DenableManagementApi=true \ + -DskipStartupExtensions=false \ + -Dautomation.mode.seq.car.name="$CAR_NAME" \ + -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \ + -Dorg.ops4j.pax.logging.logReaderEnabled=false \ + -Dorg.ops4j.pax.logging.eventAdminEnabled=false \ + -Djsch.server_host_key=ssh-rsa,ssh-dss \ + -Djsch.client_pubkey=ssh-rsa,ssh-dss \ + $NODE_PARAMS \ + -Dorg.apache.activemq.SERIALIZABLE_PACKAGES="*" \ + org.wso2.micro.integrator.bootstrap.Bootstrap $* + status=$? +done From 0a863ce6896edc85fa0b91e3216e075912e61331 Mon Sep 17 00:00:00 2001 From: chanikag Date: Tue, 7 Nov 2023 09:47:57 +0530 Subject: [PATCH 41/47] Fix testMessageProcessorCronForwader intermittent failures Fix intermittent test failure --- .../carbon/esb/jms/transport/test/MSMPCronForwarderCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/mediation-tests/tests-transport/src/test/java/org/wso2/carbon/esb/jms/transport/test/MSMPCronForwarderCase.java b/integration/mediation-tests/tests-transport/src/test/java/org/wso2/carbon/esb/jms/transport/test/MSMPCronForwarderCase.java index e35ead591e..0a652bed65 100644 --- a/integration/mediation-tests/tests-transport/src/test/java/org/wso2/carbon/esb/jms/transport/test/MSMPCronForwarderCase.java +++ b/integration/mediation-tests/tests-transport/src/test/java/org/wso2/carbon/esb/jms/transport/test/MSMPCronForwarderCase.java @@ -78,6 +78,7 @@ public void testMessageProcessorCronForwader() throws Exception { assertEquals(response4.getResponseCode(), 202, "ESB failed to send 202 even after setting FORCE_SC_ACCEPTED"); // WAIT FOR THE MESSAGE PROCESSOR TO TRIGGER + Thread.sleep(60000); Awaitility.await().pollInterval(50, TimeUnit.MILLISECONDS).atMost(60, TimeUnit.SECONDS).until(isLogWritten()); assertTrue(carbonLogReader.checkForLog("Jack", 60, NUMBER_OF_MESSAGES)); } From 5277424b12f669ac6b0e8f1ba0514f6c34e6c36f Mon Sep 17 00:00:00 2001 From: chanikag Date: Thu, 9 Nov 2023 10:38:45 +0530 Subject: [PATCH 42/47] Fix intermittent test failure $subject --- .../test/EI1622JMSInboundMessagePollingConsumerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/jms/transport/test/EI1622JMSInboundMessagePollingConsumerTest.java b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/jms/transport/test/EI1622JMSInboundMessagePollingConsumerTest.java index 8d4c6c8ba4..2967be5bc2 100644 --- a/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/jms/transport/test/EI1622JMSInboundMessagePollingConsumerTest.java +++ b/integration/mediation-tests/tests-patches/src/test/java/org/wso2/carbon/esb/jms/transport/test/EI1622JMSInboundMessagePollingConsumerTest.java @@ -50,7 +50,7 @@ public void testPollingWithSuspensionLimit() throws Exception { pushMessageToQue(addEndpoint()); assertTrue(Utils.checkForLog(carbonLogReader, "Suspending polling as the pollingSuspensionLimit of 2 " - + "reached. Polling will be re-started after 3000 milliseconds", DEFAULT_TIMEOUT), + + "reached. Polling will be re-started after 3000 milliseconds", 180), "JMS Polling suspension is not enabled."); Utils.undeploySynapseConfiguration(ENDPOINT_NAME, Utils.ArtifactType.INBOUND_ENDPOINT.getDirName(), false); } @@ -61,7 +61,7 @@ public void testPollingWithSuspensionLimitAsZero() throws Exception { pushMessageToQue(addEndpointWithSuspensionLimitZero()); - assertTrue(Utils.checkForLog(carbonLogReader, "Polling is suspended permanently", DEFAULT_TIMEOUT), + assertTrue(Utils.checkForLog(carbonLogReader, "Polling is suspended permanently", 180), "JMS Polling is not permanently suspended though the suspension limit is 0."); Utils.undeploySynapseConfiguration(ENDPOINT_NAME, Utils.ArtifactType.INBOUND_ENDPOINT.getDirName(), false); } From 8fe154c21b6ffc8ae579c1c5b5d0864424dd9998 Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Thu, 9 Nov 2023 14:49:25 +0530 Subject: [PATCH 43/47] Upgrade guava version Upgrade guava version along with the other required upgrades. Exclude asm from nashorn to avoid jacoco issues in tests. --- .../pom.xml | 3 --- .../mediation-tests/tests-mediator-2/pom.xml | 14 +++++----- pom.xml | 26 ++++--------------- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/features/org.wso2.micro.integrator.server.feature/pom.xml b/features/org.wso2.micro.integrator.server.feature/pom.xml index 502e0d720d..8440fc3a15 100644 --- a/features/org.wso2.micro.integrator.server.feature/pom.xml +++ b/features/org.wso2.micro.integrator.server.feature/pom.xml @@ -276,9 +276,6 @@ commons-configuration:commons-configuration:${commons.configuration.version} org.wso2.orbit.io.swagger.v3:swagger-parser:${swagger.parser.orbit.version} - io.swagger:swagger-core:${swagger.core.v2.version} - io.swagger:swagger-models:${swagger.core.v2.version} - io.swagger:swagger-annotations:${swagger.core.v2.version} com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${fasterxml.jackson.version} javax.validation:validation-api:${javax.validation.version} org.yaml:snakeyaml:${snakeyaml.version} diff --git a/integration/mediation-tests/tests-mediator-2/pom.xml b/integration/mediation-tests/tests-mediator-2/pom.xml index 630120bd26..bc3d92bda3 100644 --- a/integration/mediation-tests/tests-mediator-2/pom.xml +++ b/integration/mediation-tests/tests-mediator-2/pom.xml @@ -417,14 +417,12 @@ org.openjdk.nashorn nashorn-core - - - org.ow2.asm - asm-util - - - org.ow2.asm - asm-commons + + + org.ow2.asm + asm + + diff --git a/pom.xml b/pom.xml index 894d2020aa..debef74b99 100644 --- a/pom.xml +++ b/pom.xml @@ -1276,21 +1276,6 @@ swagger-parser ${swagger.parser.orbit.version} - - io.swagger - swagger-core - ${swagger.core.v2.version} - - - io.swagger - swagger-models - ${swagger.core.v2.version} - - - io.swagger - swagger-annotations - ${swagger.core.v2.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -1558,9 +1543,9 @@ 2.1.4.Final.wso2v1 - 4.0.0-wso2v57 + 4.0.0-wso2v61 [4.0.0, 4.0.1) - 4.7.175 + 4.7.182 1.1.3 [1.1.0,1.2.0) 4.5.2 @@ -1673,7 +1658,7 @@ 1.2.4 4.2.0 2.4.11 - 31.0.1-jre + 32.1.3-jre 2.1.0.wso2v1 4.8.2 4.1.86.Final @@ -1865,8 +1850,7 @@ 0.8.1 3.14.0 - 2.0.30.wso2v1 - 1.6.5 + 2.0.30.wso2v2 2.2 1.1.0.Final 3.4.2 @@ -1882,7 +1866,7 @@ 9.2 1.15.3 1.27.2.wso2v1 - 0.30.0.wso2v1 + 0.30.0.wso2v2 From e61e8412369232c7b5cfb68f7d7b09ba3d5828cf Mon Sep 17 00:00:00 2001 From: Arunan Date: Fri, 10 Nov 2023 14:02:13 +0530 Subject: [PATCH 44/47] Upgrade dependencies --- integration/pom.xml | 2 +- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/pom.xml b/integration/pom.xml index 69de98222f..8a9d6ce4d7 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -809,7 +809,7 @@ 1.9 1.1.1 3.0 - 4.1.79.Final + 4.1.100.Final 2.0.53.Final 1.2.0 5.9.1 diff --git a/pom.xml b/pom.xml index debef74b99..531a7e2050 100644 --- a/pom.xml +++ b/pom.xml @@ -1590,7 +1590,7 @@ 3.17.0.wso2v1 3.0.0.wso2v1 1.1.400.v20130418-1354 - 9.0.70 + 9.0.82 ${version.tomcat}.wso2v1 ${version.tomcat}.wso2v1 ${version.tomcat}.wso2v1 @@ -1661,7 +1661,7 @@ 32.1.3-jre 2.1.0.wso2v1 4.8.2 - 4.1.86.Final + 4.1.100.Final 2.0.53.Final 0.8.8 5.2.0 @@ -1831,7 +1831,7 @@ 8.0.19 10.3.2.1wso2v1 - 9.0.58.wso2v1 + 9.0.82.wso2v1 [1.7.0,2.0.0) 4.5.1.wso2v1 From ebb5f96c9f519ca3c8d803e6aab7ce157dbc8936 Mon Sep 17 00:00:00 2001 From: GDLMadushanka Date: Fri, 10 Nov 2023 17:13:35 +0530 Subject: [PATCH 45/47] Add changes for the new orbit bundles Add changes for the new orbit bundles. Related to guava upgrade --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index debef74b99..0f823acc8f 100644 --- a/pom.xml +++ b/pom.xml @@ -1543,7 +1543,7 @@ 2.1.4.Final.wso2v1 - 4.0.0-wso2v61 + 4.0.0-wso2v62 [4.0.0, 4.0.1) 4.7.182 1.1.3 @@ -1844,13 +1844,13 @@ 3.2.3 9.22.0.wso2v1 [9.22,10.0) - 1.21.0 + 1.59.0 1.6.2 0.8.1 3.14.0 - 2.0.30.wso2v2 + 2.1.18.wso2v1 2.2 1.1.0.Final 3.4.2 @@ -1865,8 +1865,8 @@ 15.3 9.2 1.15.3 - 1.27.2.wso2v1 - 0.30.0.wso2v2 + 1.59.0.wso2v1 + 0.31.1.wso2v2 From 1a9209fe804df98104bc8def01d9969154007b5d Mon Sep 17 00:00:00 2001 From: Arunan Date: Sat, 11 Nov 2023 14:37:40 +0530 Subject: [PATCH 46/47] Add correct jdbc pool version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e137a1d166..dd6b32df10 100644 --- a/pom.xml +++ b/pom.xml @@ -1831,7 +1831,7 @@ 8.0.19 10.3.2.1wso2v1 - 9.0.82.wso2v1 + 9.0.58.wso2v1 [1.7.0,2.0.0) 4.5.1.wso2v1 From 3a04d7d96a68ac3d4f541823fef921a92118fe44 Mon Sep 17 00:00:00 2001 From: Arunan Date: Mon, 20 Nov 2023 13:39:10 +0530 Subject: [PATCH 47/47] Update cluster mssql script to avoid warnings --- .../src/dbscripts/mssql/mssql_cluster.sql | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/distribution/src/dbscripts/mssql/mssql_cluster.sql b/distribution/src/dbscripts/mssql/mssql_cluster.sql index 2ec6d7d1d8..b8c304ba4f 100644 --- a/distribution/src/dbscripts/mssql/mssql_cluster.sql +++ b/distribution/src/dbscripts/mssql/mssql_cluster.sql @@ -1,15 +1,15 @@ IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[LEADER_STATUS_TABLE]') AND TYPE IN (N'U')) CREATE TABLE LEADER_STATUS_TABLE ( - GROUP_ID VARCHAR (512) NOT NULL, - NODE_ID VARCHAR (512) NOT NULL, + GROUP_ID VARCHAR (450) NOT NULL, + NODE_ID VARCHAR (450) NOT NULL, LAST_HEARTBEAT BIGINT NOT NULL, PRIMARY KEY (GROUP_ID) ); IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[CLUSTER_NODE_STATUS_TABLE]') AND TYPE IN (N'U')) CREATE TABLE CLUSTER_NODE_STATUS_TABLE ( - GROUP_ID VARCHAR (512) NOT NULL, - NODE_ID VARCHAR (512) NOT NULL, + GROUP_ID VARCHAR (450) NOT NULL, + NODE_ID VARCHAR (450) NOT NULL, IS_NEW_NODE INT NOT NULL, LAST_HEARTBEAT BIGINT NOT NULL, PRIMARY KEY (GROUP_ID, NODE_ID) @@ -17,23 +17,23 @@ CREATE TABLE CLUSTER_NODE_STATUS_TABLE ( IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[MEMBERSHIP_EVENT_TABLE]') AND TYPE IN (N'U')) CREATE TABLE MEMBERSHIP_EVENT_TABLE ( - GROUP_ID VARCHAR (512) NOT NULL, - NODE_ID VARCHAR (512) NOT NULL, + GROUP_ID VARCHAR (450) NOT NULL, + NODE_ID VARCHAR (450) NOT NULL, CHANGE_TYPE INT NOT NULL, - CHANGED_MEMBER_ID VARCHAR (512) NOT NULL + CHANGED_MEMBER_ID VARCHAR (450) NOT NULL ); IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[REMOVED_MEMBERS_TABLE]') AND TYPE IN (N'U')) CREATE TABLE REMOVED_MEMBERS_TABLE ( - GROUP_ID VARCHAR (512) NOT NULL, - NODE_ID VARCHAR (512) NOT NULL, - REMOVED_MEMBER_ID VARCHAR (512) NOT NULL + GROUP_ID VARCHAR (450) NOT NULL, + NODE_ID VARCHAR (450) NOT NULL, + REMOVED_MEMBER_ID VARCHAR (450) NOT NULL ); IF NOT EXISTS (SELECT * FROM SYS.OBJECTS WHERE OBJECT_ID = OBJECT_ID(N'[DBO].[COORDINATED_TASK_TABLE]') AND TYPE IN (N'U')) CREATE TABLE COORDINATED_TASK_TABLE ( TASK_NAME VARCHAR (512) NOT NULL, - DESTINED_NODE_ID VARCHAR (512), + DESTINED_NODE_ID VARCHAR (450), TASK_STATE VARCHAR (32), PRIMARY KEY (TASK_NAME) );