From 58579384efd183d37c1950cdc3cc7608d6b8cf5a Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Mon, 5 Jun 2023 11:18:23 +0200 Subject: [PATCH 001/114] Add skeleton --- plugins/mqtt-interface/build.gradle | 70 +++++++++ plugins/mqtt-interface/gradle.properties | 27 ++++ .../db/mqtt/MqttInterfacePlugin.java | 146 ++++++++++++++++++ settings.gradle | 1 + 4 files changed, 244 insertions(+) create mode 100644 plugins/mqtt-interface/build.gradle create mode 100644 plugins/mqtt-interface/gradle.properties create mode 100644 plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java diff --git a/plugins/mqtt-interface/build.gradle b/plugins/mqtt-interface/build.gradle new file mode 100644 index 0000000000..35361d355e --- /dev/null +++ b/plugins/mqtt-interface/build.gradle @@ -0,0 +1,70 @@ +group "org.polypheny" + + +dependencies { + compileOnly project(":core") + compileOnly project(":monitoring") + + + // --- Test Compile --- + testImplementation project(path: ":core", configuration: "tests") + testImplementation project(path: ":core") + testCompileOnly group: 'org.pf4j', name: 'pf4j', version: pf4jVersion + testImplementation group: "junit", name: "junit", version: junit_version +} + + +sourceSets { + main { + java { + srcDirs = ["src/main/java"] + outputDir = file(project.buildDir.absolutePath + "/classes") + } + resources { + srcDirs = ["src/main/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/classes") + } + test { + java { + srcDirs = ["src/test/java"] + outputDir = file(project.buildDir.absolutePath + "/test-classes") + } + resources { + srcDirs = ["src/test/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") + } +} + +compileJava { + dependsOn(":config:processResources") + dependsOn(":core:processResources") + dependsOn(":information:processResources") + dependsOn(":monitoring:processResources") +} + +delombok { + dependsOn(":monitoring:processResources") +} + + +/** + * JARs + */ +jar { + manifest { + attributes "Manifest-Version": "1.0" + attributes "Copyright": "The Polypheny Project (polypheny.org)" + attributes "Version": "$project.version" + } +} +java { + withJavadocJar() + withSourcesJar() +} + +licensee { + allow('Apache-2.0') + allow('MIT') +} diff --git a/plugins/mqtt-interface/gradle.properties b/plugins/mqtt-interface/gradle.properties new file mode 100644 index 0000000000..6140d42ef2 --- /dev/null +++ b/plugins/mqtt-interface/gradle.properties @@ -0,0 +1,27 @@ +# +# Copyright 2019-2023 The Polypheny Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pluginVersion = 0.0.1 + +pluginId = mqtt-interface +pluginClass = org.polypheny.db.mqtt.MqttInterfacePlugin +pluginProvider = The Polypheny Project +pluginDependencies = +pluginUrlPath = +pluginCategories = interface +pluginPolyDependencies = +pluginIsSystemComponent = false +pluginIsUiVisible = true \ No newline at end of file diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java new file mode 100644 index 0000000000..30080bf278 --- /dev/null +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java @@ -0,0 +1,146 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + + +import com.google.common.collect.ImmutableList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.pf4j.Extension; +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.polypheny.db.iface.Authenticator; +import org.polypheny.db.iface.QueryInterface; +import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.information.InformationManager; +import org.polypheny.db.transaction.TransactionManager; + + +public class MqttInterfacePlugin extends Plugin { + + + /** + * Constructor to be used by plugin manager for plugin instantiation. + * Your plugins have to provide constructor with this exact signature to be successfully loaded by manager. + */ + public MqttInterfacePlugin( PluginWrapper wrapper ) { + super( wrapper ); + } + + + @Override + public void start() { + // Add REST interface + Map mqttSettings = new HashMap<>(); + mqttSettings.put( "broker", "localhost" ); + mqttSettings.put( "brokerPort", "5555" ); + QueryInterfaceManager.addInterfaceType( "mqtt", MqttInterfaceServer.class, mqttSettings ); + } + + + @Override + public void stop() { + QueryInterfaceManager.removeInterfaceType( MqttInterfaceServer.class ); + } + + + @Slf4j + @Extension + public static class MqttInterfaceServer extends QueryInterface { + + @SuppressWarnings("WeakerAccess") + public static final String INTERFACE_NAME = "MQTT Interface"; + @SuppressWarnings("WeakerAccess") + public static final String INTERFACE_DESCRIPTION = "ADD TEXT."; + @SuppressWarnings("WeakerAccess") + public static final List AVAILABLE_SETTINGS = ImmutableList.of( + new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), + new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 5555 ), + new QueryInterfaceSettingString( "topics", false, true, true, null ) + ); + + private final String uniqueName; + + private final String broker; + + private final MonitoringPage monitoringPage; + + + public MqttInterfaceServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { + super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); + //this.requestParser = new RequestParser( transactionManager, authenticator, "pa", "APP" ); + this.uniqueName = uniqueName; + // Add information page + monitoringPage = new MonitoringPage(); + broker = settings.get( "broker" ); + } + + + @Override + public void run() { + log.info( "{} started", INTERFACE_NAME ); + } + + + @Override + public List getAvailableSettings() { + return AVAILABLE_SETTINGS; + } + + + @Override + public void shutdown() { + //restServer.stop(); + //monitoringPage.remove(); + log.info( "{} stopped.", INTERFACE_NAME ); + } + + + @Override + protected void reloadSettings( List updatedSettings ) { + // There is no modifiable setting for this query interface + } + + + @Override + public void languageChange() { + + } + + + @Override + public String getInterfaceType() { + return INTERFACE_NAME; + } + + + private class MonitoringPage { + + + public MonitoringPage() { + InformationManager im = InformationManager.getInstance(); + } + + + } + + } + +} + diff --git a/settings.gradle b/settings.gradle index e86ba4affb..3b3603f18a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,7 @@ include 'plugins:jdbc-adapter-framework' include 'plugins:avatica-interface' include 'plugins:rest-interface' include 'plugins:http-interface' +include 'plugins:mqtt-interface' // adapters plugins include 'plugins:hsqldb-adapter' From ddb0b5836b7d2ba366992fe97d1ff62573ebaf28 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 12 Jun 2023 09:42:20 +0200 Subject: [PATCH 002/114] Added mqtt client dependency to gradle --- gradle.properties | 1 + plugins/mqtt-interface/build.gradle | 3 +++ 2 files changed, 4 insertions(+) diff --git a/gradle.properties b/gradle.properties index 9550469503..0131529122 100644 --- a/gradle.properties +++ b/gradle.properties @@ -47,6 +47,7 @@ commons_text_version = 1.10.0 cottontaildb_version = 0.13.3 cottontaildb_driver_version = 0.13.0 cottontaildb_grpc_version = 1.36.0 +eclipse_mqttv3_version = 1.2.5 elasticsearch_rest_client_version = 6.2.4 elasticsearch_version = 6.2.4 embedded_monetdb_version = 2.39 diff --git a/plugins/mqtt-interface/build.gradle b/plugins/mqtt-interface/build.gradle index 35361d355e..88b82db2ac 100644 --- a/plugins/mqtt-interface/build.gradle +++ b/plugins/mqtt-interface/build.gradle @@ -5,6 +5,9 @@ dependencies { compileOnly project(":core") compileOnly project(":monitoring") + // https://mvnrepository.com/artifact/org.eclipse.paho/org.eclipse.paho.client.mqttv3 + implementation group: 'org.eclipse.paho', name: 'org.eclipse.paho.client.mqttv3', version: eclipse_mqttv3_version + // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") From 1ccaa44a07f977142978bbc161eda52a39ae7b92 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 13 Jun 2023 11:30:04 +0200 Subject: [PATCH 003/114] coded the connection establishment to broker implemented the run method --- .../db/mqtt/MqttInterfacePlugin.java | 74 ++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java index 30080bf278..8fd4ba3eaf 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java @@ -22,6 +22,13 @@ import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; +import org.eclipse.paho.client.mqttv3.IMqttToken; +import org.eclipse.paho.client.mqttv3.IMqttActionListener; +import org.eclipse.paho.client.mqttv3.MqttCallback; +import org.eclipse.paho.client.mqttv3.MqttException; +import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.eclipse.paho.client.mqttv3.MqttPersistenceException; import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; @@ -30,6 +37,8 @@ import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.information.InformationManager; import org.polypheny.db.transaction.TransactionManager; +import org.eclipse.paho.client.mqttv3.MqttAsyncClient; +import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; public class MqttInterfacePlugin extends Plugin { @@ -67,18 +76,20 @@ public static class MqttInterfaceServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_NAME = "MQTT Interface"; @SuppressWarnings("WeakerAccess") - public static final String INTERFACE_DESCRIPTION = "ADD TEXT."; + public static final String INTERFACE_DESCRIPTION = "Connection establishment to MQTT broker."; @SuppressWarnings("WeakerAccess") public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 5555 ), - new QueryInterfaceSettingString( "topics", false, true, true, null ) + new QueryInterfaceSettingList( "topics", false, true, true, null ) ); private final String uniqueName; private final String broker; + private final String brokerPort; + private final MonitoringPage monitoringPage; @@ -89,12 +100,69 @@ public MqttInterfaceServer( TransactionManager transactionManager, Authenticator // Add information page monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); + brokerPort = settings.get( "brokerPort" ); } @Override public void run() { - log.info( "{} started", INTERFACE_NAME ); + String serverURI = String.format( "tcp://%s:%s", broker, brokerPort); + // creating fileStore to store all messages in this directory folder + MqttDefaultFilePersistence fileStore = new MqttDefaultFilePersistence("C:\\Users\\Public\\UniProjekte\\BA_MQTT_Messages"); + try { + fileStore.open(uniqueName, serverURI); + } catch (MqttPersistenceException e) { + log.error( "There is a problem reading or writing persistence data." ); + } + try { + MqttAsyncClient client = new MqttAsyncClient(serverURI, uniqueName, fileStore); + MqttCallback callback = new MqttCallback() { + @Override + public void connectionLost(Throwable cause) { + log.error( "Lost connection to the broker!" ); + //TODO: show this on UI! + } + + @Override + public void messageArrived(String topic, MqttMessage message) throws Exception { + log.info( "Message: {}", message.toString()); + //TODO: extract the topic content of the message + // AND send it to StreamProcessor as PolyStream. + } + + @Override + public void deliveryComplete(IMqttDeliveryToken token) { + log.info( "Delivery of Message was successful!" ); + } + }; + + client.setCallback(callback); + IMqttActionListener connectionListener = new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + log.info( "{} started and is listening to broker {}:{}", INTERFACE_NAME, broker, brokerPort ); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); + } + }; + IMqttToken connToken = client.connect(null, connectionListener); + connToken.waitForCompletion(); + + + // Testing the connection: + String str = "Hello, I am the Polypheny-Client!"; + MqttMessage msg = new MqttMessage(str.getBytes()); + IMqttToken pubToken= client.publish("testTopic", msg); + + // TEsting subscribtion: + IMqttToken subToken= client.subscribe("testTopic", 1); + + } catch (MqttException e) { + log.error( "An error occurred while communicating to the server."); + } } From 73ab690aebbab90f52128625dad99d0571ffeab2 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 16 Jun 2023 16:34:00 +0200 Subject: [PATCH 004/114] Implemented shutdown method --- .../db/mqtt/MqttInterfacePlugin.java | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java index 8fd4ba3eaf..84eb7a2f04 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java @@ -80,7 +80,7 @@ public static class MqttInterfaceServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), - new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 5555 ), + new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), new QueryInterfaceSettingList( "topics", false, true, true, null ) ); @@ -90,6 +90,8 @@ public static class MqttInterfaceServer extends QueryInterface { private final String brokerPort; + private MqttAsyncClient client; + private final MonitoringPage monitoringPage; @@ -108,6 +110,7 @@ public MqttInterfaceServer( TransactionManager transactionManager, Authenticator public void run() { String serverURI = String.format( "tcp://%s:%s", broker, brokerPort); // creating fileStore to store all messages in this directory folder + //won't be needed later, when data is stored in data store MqttDefaultFilePersistence fileStore = new MqttDefaultFilePersistence("C:\\Users\\Public\\UniProjekte\\BA_MQTT_Messages"); try { fileStore.open(uniqueName, serverURI); @@ -115,7 +118,7 @@ public void run() { log.error( "There is a problem reading or writing persistence data." ); } try { - MqttAsyncClient client = new MqttAsyncClient(serverURI, uniqueName, fileStore); + client = new MqttAsyncClient(serverURI, uniqueName, fileStore); MqttCallback callback = new MqttCallback() { @Override public void connectionLost(Throwable cause) { @@ -126,7 +129,9 @@ public void connectionLost(Throwable cause) { @Override public void messageArrived(String topic, MqttMessage message) throws Exception { log.info( "Message: {}", message.toString()); - //TODO: extract the topic content of the message + MqttDocumentStore store = new MqttDocumentStore(); + store.saveMessage(topic, message); + //TODO: extract the important content of the message // AND send it to StreamProcessor as PolyStream. } @@ -140,11 +145,13 @@ public void deliveryComplete(IMqttDeliveryToken token) { IMqttActionListener connectionListener = new IMqttActionListener() { @Override public void onSuccess(IMqttToken asyncActionToken) { + //TODO: show on UI log.info( "{} started and is listening to broker {}:{}", INTERFACE_NAME, broker, brokerPort ); } @Override public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + //TODO: show on UI log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); } }; @@ -153,19 +160,18 @@ public void onFailure(IMqttToken asyncActionToken, Throwable exception) { // Testing the connection: - String str = "Hello, I am the Polypheny-Client!"; - MqttMessage msg = new MqttMessage(str.getBytes()); - IMqttToken pubToken= client.publish("testTopic", msg); + //String str = "Hello, I am the Polypheny-Client!"; + //MqttMessage msg = new MqttMessage(str.getBytes()); + //IMqttToken pubToken= client.publish("testTopic", msg); - // TEsting subscribtion: - IMqttToken subToken= client.subscribe("testTopic", 1); + // Testing subscribtion: + //IMqttToken subToken= client.subscribe("testTopic", 1); } catch (MqttException e) { log.error( "An error occurred while communicating to the server."); } } - @Override public List getAvailableSettings() { return AVAILABLE_SETTINGS; @@ -174,9 +180,28 @@ public List getAvailableSettings() { @Override public void shutdown() { - //restServer.stop(); + IMqttActionListener shutDownListener = new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + //TODO: show on UI + log.info( "{} stopped.", INTERFACE_NAME); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + //TODO: show on UI + log.info( "{} could not disconnected from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + } + }; + + try { + client.disconnect(null, shutDownListener); + } catch (MqttException e) { + log.error( "An error occurred while disconnecting from the broker {}:{}. Please try again.", broker, brokerPort ); + } + //monitoringPage.remove(); - log.info( "{} stopped.", INTERFACE_NAME ); + } From 24936a26dae2143c7fd81178e293dbeca7ff28f4 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 16 Jun 2023 17:58:28 +0200 Subject: [PATCH 005/114] Implemented sub, unsub and pub methods methods can't be triggered yet. --- .../db/mqtt/MqttInterfacePlugin.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java index 84eb7a2f04..ecda5df655 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java @@ -90,6 +90,8 @@ public static class MqttInterfaceServer extends QueryInterface { private final String brokerPort; + private List topics; + private MqttAsyncClient client; private final MonitoringPage monitoringPage; @@ -205,6 +207,77 @@ public void onFailure(IMqttToken asyncActionToken, Throwable exception) { } + public void subscribe(String topic){ + //TODO: trigger from UI. + if (!topics.contains(topic)) { + IMqttActionListener subListener = new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + topics.add(topic); + log.info( "Successfull subscription to {}.", topic ); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + log.info( "Subscription was not successfull. Please try again." ); + } + }; + try { + IMqttToken subToken= client.subscribe(topic, 1, null, subListener); + } catch (MqttException e) { + log.error( "An error occurred while subscribing to {}. Please try again.", topic ); + } + } + } + + public void unsubscribe(String topic) { + //TODO: trigger from UI. + if (topics.contains(topic)) { + IMqttActionListener unsubListener = new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + topics.remove(topic); + log.info( "Successfull unsubscription from {}.", topic ); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + log.info( "Unsubscription was not successfull. Please try again." ); + } + }; + try { + IMqttToken subToken= client.unsubscribe(topic, null, unsubListener); + } catch (MqttException e) { + log.error( "An error occurred while unsubscribing from {}. Please try again.", topic ); + } + } + } + + public void publish(String topic, String msg) { + //TODO: trigger from UI. + if (topics.contains(topic)) { + IMqttActionListener pubListener = new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + log.info( "Message was published successfully."); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + log.info( "Unsubscription was not successfull. Please try again." ); + } + }; + MqttMessage message = new MqttMessage(msg.getBytes()); + try { + IMqttToken subToken= client.publish(topic, message, null, pubListener); + } catch (MqttException e) { + log.error( "An error occurred while unsubscribing from {}. Please try again.", topic ); + } + } + } + + + @Override protected void reloadSettings( List updatedSettings ) { // There is no modifiable setting for this query interface From 2ccdf625e26055efa412c8bdfe00a2829661890c Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 17 Jun 2023 11:25:29 +0200 Subject: [PATCH 006/114] Implemented MonitoringPage created table to display the subscribed topics. also fixed bug in subscribe method. --- ca-key.pem | 54 ++++++++++++++++++ ca.pem | 37 ++++++++++++ ca.srl | 1 + cert.pem | 32 +++++++++++ key.pem | 51 +++++++++++++++++ .../polypheny/db/mqtt/MqttDocumentStore.java | 39 +++++++++++++ .../db/mqtt/MqttInterfacePlugin.java | 57 +++++++++++++++++-- server-cert.pem | 33 +++++++++++ server-key.pem | 51 +++++++++++++++++ 9 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 ca-key.pem create mode 100644 ca.pem create mode 100644 ca.srl create mode 100644 cert.pem create mode 100644 key.pem create mode 100644 plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java create mode 100644 server-cert.pem create mode 100644 server-key.pem diff --git a/ca-key.pem b/ca-key.pem new file mode 100644 index 0000000000..c7db71eed3 --- /dev/null +++ b/ca-key.pem @@ -0,0 +1,54 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,DFDC177D1AF280A6464E4CE7508715C2 + +E2AnAgLZLPKO86OS68f3azVfyU0j1nkMaNvCJA+fTxJhl24gairOevgwRwScEVVJ +c3gCT2d8wrnwSJiuhCg0MIas5Wl0xD+JNI3ED0x0qd/BxccERb3Ie1+aC0n8VG0N +E1SMJ5BNhp2T/eAHSRzVV3y5yv7mnZ24sz0sjkhTVvc6X8R4kMxRVfVXUItfmPhv +CWrl4eC2iwizGXZq6f32wqhvJN8MIzltKxuwlUFTZ2RFdxSzLr3X4bJzc8b94LtN +HxnJBFWPrSxgIDVi1Df7WyM6XKURgjBHuedVwV4PYUdsRc/mx+IQAZY/kwHcsPSa +AlmlitOOP6MM4CtPDj2YB+Y5XmXwXawemAIsG+TNTbFVbYZ6YaZSjoM943D7ufvu +7zwxBkXqh+BC8dMLYtYcvkUI5uYhXhPAFmBkzfnsHccLO7qi2MDNWWjn0k52ufmN +mb4ZHAcWS66RrPtqm1fr6I+yBbtWsF8F7McNiuMtQ4qQTg8BFM1OketnhK50IQLr +BSsGEftd7DbBXyqRMKTJ2hcnrZZ/Lbt258KOUDKcdnM27xSYg5mZX8JuynV778hb +YhcGRAyO8K6Nc5X8iFsxL6xbLlL0aJaQE6RHRnr6yLDNt8R4K2VfIwh5/snzGb8d +4kMWVecykqACqf32EZPgQy4Rt/Qq9pF6aI9pS4DJ9yHtP/UxrLZ40ueiq8qbjpDJ +TJoLMmHKx4nPcjD+a0/uhvmzYft6EOZeLVpe8NlIAf8cEsw3nWfE1LaiVpAUxZaT +qSHbqQNSEeo8hwU5UGpCC1uLj/x/olfyN2DNZJZOa8J8x3+SDWH7u3MGiaw+mTE0 +hIwt/UOD83Lgygp4RDOYmmFKM33PhBm6V8Ejuwa6JKqVz/WjUfJQ/4XP8IjlOcTd +4P41OWFPnoeXULBgBF4TRd/7L/k9UPScXSitfZb3LFnkv8edH3a0jcbM+FRjZymY +LKvjqiBUbK0tk+SNqjlHSjBWy97V13qdVTgrkq5wCJ7FTtcOvdR8XoIy67UyLFva +C9kg8xCh8MNLezKCm2+tfeDVD0HXhSfaw+8yiNbg4k8r2dIQDRf91I90MH/BZlE4 +fOwS9DrMxJxNzjUuM2PWnllr7vubfn9n4jMw1IX2giTVlYj6CFCP5bsGGeeawlk5 +uFaYakTdfiwDW3YfWI2g/n3I+us1BFp2RivzJYehXFhgDfDDASOROU/QSB6pkEn4 +GylWbvoP9qTS+V76rS2uhGsfSBJhu9u4TYiplpBjtvgiCXsjuTb7m0nNv0tyGiDH +BKRdU2JqAINjHGl9H8qjT7KKrkDF+uib7qBpXoQN+Q++CwitzoPwKUFS+6d3R2Bl +eHfaGYvAdYV3QrmKCnUGZdmNVTYARxTFdiKEEHoSzrOrj3puB13BWSxcB7KjaMQl +/wkNF9DCLpRzPMPRkCgWnFCncDMoeA5z8AmwBu1b4z/pAP+RItc9ClmJv+xgl27v +v1FWNxHFMBktzDXUPg3sYXUVQ1p89qzkM8001t4fU7HsDuczNt0t6TEx9xlypU4w +FMI6htsQofRkMqAvquLET98hMS6AC1p309V2gxpE9bgvrZWtAd2jt4JVqyl9EN98 +UwmJQL08k9c2bApLZSX0af895xTH/PGdBLFzjOgnBBrQ6F4D5zUls6FZIhfnFKXM +ATQmbZriaohQAGlblzV1UlWa/x0xpjeyvdv9eCj5G06SteLm7w+gJyU0AZgcqYcx +NAcCa8x+auYQOMQlDk901yu9/iKFntKchL5EwUIp8+vTGqWAX/8OS44ULIfqoO5/ +ZpCL1Bc0XbXWKrCO1NpZj16lCYuz5Noajx+kZZsWbKubCy6BU1Fu+JUUYGTkZSPF +AWgPNLNKHfbX+0PkdXv8zmw7OV21JqOSy6eEqeQ4zY/+k6irTpPz+DBnT7AQdvZ5 +ZBnFU3AdjLwtKe1xjgteBygYT9d0SAkX5N8nov7tnt/TDpVPcJU4abPH/PRfAXCe +wNniIfK8nQWfjjHeZmS4+hX+oqWTnHynKECzmnIDm0oqY2h4Tgg8DHwwfCC6Oot6 +0sAvcdSF2csG5E/yr3K3xjQKh73Nf+NdggTJFypba/bEwZgdSoaJM63agh62GNuC +d8KphnJy9OQI3fwp5ftcpjJgHVuBaK06bMR6gAMWCYH3iomRqU9cLUt8/X4DI/6d +M8BA1ZBiJ9s922GSzQelWaONcqOCb0k20IaAI1rwjCJqrWCl3p5z1E/3X1gKhAPQ +5e2dQJo5qctDLi8nHin8IZf3GD9iannzzUxBAQhKP7HSDY5lNcP6d3qpHilKpccq +kCOMju1JDDaUfcuzbkDIEJ9E/eDPcgXX+ukC6Z++Osy398NE4RYagKl5RXZtuLXL +s32Q281H6KkOCG+V5OccGqi5EO6AzP+GAE6P61jDqWdynLzUgRqX4Sgxr6K4zVrt +TnAFG1X9NwXqLrfMvLD/lMR5aAkY41gUwBT0AF+bWGHMD7j5eNNI8cZLBOk1cKa5 +4bfhE/mkJ37vPxQa3Vx+03fyr3T4k1iUlqiPjnHZ7CaJV1BBDlWJt87zDUlHJyd0 +1SUMQ/6sFTB0oXNN3uW1V98l/tT+7iJ7orGCFwrdRxgbXsc+sdOJtEoUfD01vIns +txGxEf4bQlcZi2pQkJe8Ov4Djzsjyf8fYZ8bRLhkRaiVMvUPvJXv1AvfDFS7Y2Vh +BdWJtJMkRv10olxcRzzL1uoH562QpYuUjoscblHMBP1SWVODy5ou/+c2xBefg59N +71On/7DPC3OO+TAbirNR3nEsAVwmlSsygB/7l0xI8Q6tcFwJx7NgIcvy9mW/fTW3 +YbWC98QEGEd4Zz60+itDnAGa3vPNKR1JbD5Gez6WpJqRdY4SroATqEjiX9c03tJH +YRxW126kNmQkoTunXAwXuhAUSv1CpFQwep6Kjk0ySEfatxIbddujSveIgzIlIdIn +8XVJqJgerJ//bw/A+3ZNSn8UZYwM2hcSF2GFPIXj/cNERMVMDoffvnREvpD7yAJQ +LOJ0+uwyAiULSI8WkzLe1xGiWHaWUiOuILaIQetRsgO0rujT9qRJi9pv2ZuLY9Um +/lzoCyI/8Gyn2KF9k1O2sDcPEEQRPm1BM48O0G5eiQshREWvEyWyvIbHfIBHuCUt +-----END RSA PRIVATE KEY----- diff --git a/ca.pem b/ca.pem new file mode 100644 index 0000000000..cfc3f92fa3 --- /dev/null +++ b/ca.pem @@ -0,0 +1,37 @@ +-----BEGIN CERTIFICATE----- +MIIGYTCCBEmgAwIBAgIUeA1gR7WY00gfgt2m7y+u5ZbjlEgwDQYJKoZIhvcNAQEL +BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES +MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs +MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM +cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh +cy5jaDAeFw0yMzA1MTIxNjA0MjhaFw0yNDA1MTExNjA0MjhaMIG/MQswCQYDVQQG +EwJDSDEaMBgGA1UECAwRS2FudG9uIEJhc2VsLUxhbmQxEjAQBgNVBAcMCUFsbHNj +aHdpbDEcMBoGA1UECgwTVW5pdmVyc2l0eSBvZiBCYXNlbDEjMCEGA1UECwwaRGVw +YXJ0bWVudCBDb211dGVyIFNjaWVuY2UxFTATBgNVBAMMDHBvbHlwaGVueS1kYjEm +MCQGCSqGSIb3DQEJARYXbi5zZWx2YW5Ac3R1ZC51bmliYXMuY2gwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQDCrtrmEPaXKaKrtTklhCv/5ayoHXTdiY8j +DxubtXSgk0v+waeGNYrvmLTSX25Jvh7FEFVvdT8+A9aB9sh0SmDqLMsjDI0v/lB+ +o6aN1BhZaOxPef9lCzGdyxm3Hm5Ww76Ra1rstjRUmw9RXn5S4aFsYTwLt7s7Qds0 +QuVT9Zi1HcAcgqwRBYysaBb9I7p59LursYCcx7ZIhiuWESJNguzV00r9UGZTS56R +qOjLizJcGEuuL38iUNsNHVb0IWbW/nGgYzsKZkZMvEBPoA0Zi4x0b1EyTGrwkOQ7 +AsBho9uVkbrQwydfjrX8eoaQxM0K9m2IEZi6Sb3B1b3wW16dw1mS/zxjGHDtyogA +RCM2Ru2GwC4h8WoT2VrKgReDn0384pOjVM2m84y19S5U/6iFvtx+DqvU7T/FEaiD +8l8oGitlc4Uj2/ls1q8OxZbck1RlCNn75UbOuw4VyjNgqBvmdrdsj9Hb5ofyFm5+ +5SLtVPLKRKm9GXKmp9uo7CTx6UtX6p3LKvllW8czu9GQZnGvCn8sRnuIInhheeUp +ymUrBf64BTvosaZusIIsOziqlSrEp2YcI/8iHV+++p9SsChjU0FPALiTrZJ9SAz/ +OWqstBW0QskimW7YBYs8FPWYGTanDirGdSLbc5o2jtn3x3DFkA8BGJFs12EvcW0g +a/iQgW34uQIDAQABo1MwUTAdBgNVHQ4EFgQUX6ncnORVSzHkGNWTjWNGP3YG27sw +HwYDVR0jBBgwFoAUX6ncnORVSzHkGNWTjWNGP3YG27swDwYDVR0TAQH/BAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAgEAtM7fO11MtMk+XMf3m2nO8pgRwB1B2LyysWRW +iyzCVWvVYQJIiVU3BfLZBlCjDqcwFzSOmREX4qSfxDsR+vzEeBpxLce7DoRvJrT1 +8ab7+wUWne4/GrtJU1nZT6SuVgWpgw1qISJ2cHkBvJoR0aGu6q63nzzg1KVOe6d8 +DLShMIRDK5uRGBtKJWnyre4JSDo0PdXkflb85XkhQ9rc72LsVQ6S/t5A2kBQptk0 +Uh7Adbt1HVr0LXEujDQxMkvclLeUbgHLBCmAeD1xZjEp7X+zT+iykG1Bx0CnpVNV +/w76XsiVvLqEvnene5O3qZNmKRXhMe2X3mGHDkgtFMbRIsX0AyMiVjjnJQyJZ9Ka +8c7Nah89ldWwyy6h+A+s7OLCNvH0F1kTgwLGGTtXoF8BMqrtQ4/HRLZTNWwDBG9S +wBW2uIM/j9DtI/AxiglwtJcJZe0/Yg+B9yh9+lVij+yVIUonqe9Db7tb3QjR1PF8 +y1kaG5UbHmLEa7hiybd0J0ZhbfAlq2l8GbvqCm7GhbIljKrwnw4YKwgDc3dwTn9T +emQs12FVXSMPtsLyhsCXRKR6au5N1eKavntrn6R/njyHbB1KAAFI1UM8/xiXBq0t +lAxr7czdmMYmaUjQGuxA3czlmOZN4dPQZ0LGCyM/SNcYXAPm3Kin8kYW8RzmnNhm +S0JHkyk= +-----END CERTIFICATE----- diff --git a/ca.srl b/ca.srl new file mode 100644 index 0000000000..d2de10eb43 --- /dev/null +++ b/ca.srl @@ -0,0 +1 @@ +5822FC00F7B671C2B2052685462602F4648DDA9E diff --git a/cert.pem b/cert.pem new file mode 100644 index 0000000000..e2e4e04f6c --- /dev/null +++ b/cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFdjCCA16gAwIBAgIUWCL8APe2ccKyBSaFRiYC9GSN2p4wDQYJKoZIhvcNAQEL +BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES +MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs +MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM +cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh +cy5jaDAeFw0yMzA1MTIxNjU0MDJaFw0yNDA1MTExNjU0MDJaMBExDzANBgNVBAMM +BmNsaWVudDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALC0ab/lYM1T +pS5PgCiRHpWGib1Qy6ohIv4Cg9qUN6ghNmiatIA4O5JzyvJhK2HX3ojULNwjmost +UjhO8jjtqUULEeI8upFeeQF8ptGXJrCBeSp9ZwvB/ROheLZkZrkUFd8vTqXa2WBh +qE4SQ6xCUl5mnVx7/9mA9sS+rdvKLV3w6Hd6KEp09xeyqOBE5+tAh6JLe+QOJe75 +kHO4xqMnq0pVJJ/2fphEThCIFgTD8yMDvXaeWRLZCYhJ7F4dHCcFtrl9GPlnd7II +N0PXpEJb46QUd2Dhvg29jcsCVIo1mHWppi+MJyIgZ/2f1YrKquRTRYOaMAuc/grK +RY4Vv9iW5orfq4dZiOxrF1p4Y4NGJLuGtPOwIHoFFJZHRk0fa3AoGEvaY68/8tVm +HL8jEcnsKTzS7oQU5S7tOXOBO8A5i0YVcKaVM3YkncKfRzlLKPs6qYiSwantFYY/ +aJRABbHv0S/nCHbaaQUZV8L/SkFAsaBOz9vyW4fd25bbaohiPTR8TwLHve84hpyR +4esOfv4TEbZYy5/jdwKQHg4RnKXjWoJTbsp98CClOR+3hKyZn5SVpouak+4BMrPh +bxWjcd+nCJYBPFSOFaqDdqLKbF2kEXsYanuXXRuEuKILokXcFKNm7ra28DNKzsXj +fIn0d+1RQkYaXnX3DJsdFZEHPA/YWkbLAgMBAAGjFzAVMBMGA1UdJQQMMAoGCCsG +AQUFBwMCMA0GCSqGSIb3DQEBCwUAA4ICAQBEGt7ipiGBeHhNn8DV4SaY4uIYjti3 +B9PuSCRL6uaGpeQ8rESTBzdcpr0bHjCQvLh/ceY/w8PAGEihSzbOtfpBq9V/4pi3 +D21yiT7Z62DMl9G9bK1KD5yzsCK5llap3cBSqAVO+6rH72k3Awu/EwG3kCG/s8V4 +7nC5snml+hDos14G+NXz17EpybeaM3Ylmi2O5SJHUpByfKcX4sFOUgpHqg4miFcr +1RdSnoW2witIi8FI/LCVCXmsWU4I1KXwOFKpXFVD/dmyyohbU4gNn0KAY9lapFoF +onR0RpbmXMgS4fjnHjgSNYsofSrTW3mhLvYGy+iqAoDDvJXQMAbIKMjAgHeThnB2 +C0omXt1B2bUYB45OOFDpm3pO8b87FW8S94vnVaQ05UmINDYVZ+V/uWhXArLEaA/c +VTnZd4yJklrXBHBUv7WNSuDl3ouX5t5PfF61JX/VNu1U35Pbe8/6PFT9KNisRp45 +9gcASkJHa6MA/avIV3bcwFjxIgkExEyGwxgQ3BjCd9XkaP2HA8kEaW1jq70nLx4+ +/P3CLQOVhe0a79goYYGIRsdZrQJLIUSXwdsuBTkk1l+5UkFmfd9hBdpJfMS53hDg +vEqbbQTHlPNaLkmyVfgH0hj/YeN7b+5pyD9ioIYJOBaJe5hDstQM4Dw8e/EX6q6B +vA7FeU7gmG91ww== +-----END CERTIFICATE----- diff --git a/key.pem b/key.pem new file mode 100644 index 0000000000..52463195f7 --- /dev/null +++ b/key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAsLRpv+VgzVOlLk+AKJEelYaJvVDLqiEi/gKD2pQ3qCE2aJq0 +gDg7knPK8mErYdfeiNQs3COaiy1SOE7yOO2pRQsR4jy6kV55AXym0ZcmsIF5Kn1n +C8H9E6F4tmRmuRQV3y9OpdrZYGGoThJDrEJSXmadXHv/2YD2xL6t28otXfDod3oo +SnT3F7Ko4ETn60CHokt75A4l7vmQc7jGoyerSlUkn/Z+mEROEIgWBMPzIwO9dp5Z +EtkJiEnsXh0cJwW2uX0Y+Wd3sgg3Q9ekQlvjpBR3YOG+Db2NywJUijWYdammL4wn +IiBn/Z/Visqq5FNFg5owC5z+CspFjhW/2Jbmit+rh1mI7GsXWnhjg0Yku4a087Ag +egUUlkdGTR9rcCgYS9pjrz/y1WYcvyMRyewpPNLuhBTlLu05c4E7wDmLRhVwppUz +diSdwp9HOUso+zqpiJLBqe0Vhj9olEAFse/RL+cIdtppBRlXwv9KQUCxoE7P2/Jb +h93blttqiGI9NHxPAse97ziGnJHh6w5+/hMRtljLn+N3ApAeDhGcpeNaglNuyn3w +IKU5H7eErJmflJWmi5qT7gEys+FvFaNx36cIlgE8VI4VqoN2ospsXaQRexhqe5dd +G4S4oguiRdwUo2butrbwM0rOxeN8ifR37VFCRhpedfcMmx0VkQc8D9haRssCAwEA +AQKCAgBZJ427flXrAuryhLYawa12r9zLIfrAKdvn4Qj5vZKfR7KV8IhlsPmNmNKq +nBAxW6Mx0iuYHnaZBhLXyi04P+EJrt4lEZLoSYwwDd51HRguMgUaRQ86xXOU9gFb +tSiG9PVcGBhMYjEPCfbK/Tgbr5Kzgn6e7QlXilP3TGWX4Vgmb6g6r+2oM39/+2pt +TRbegT5tyjwGAoVtH4mceXvvUdKAvJiwBMQ2uy/P83l4ig0ge4CaWqaCdfLzSqjx +iyprHdzllZKGM4gW0UJg/mon+QjP/muE/CDR+fOSm+hfm08C/seUcdhExajyNDSf +xEWY0dLe5WqcUWdztj6Fw7953hkO3aH920hGeCaP+91zFmLfOZOVIfvpXa/C0XmY +uezb1X6JEBeHb4l1qNnOGiWf3CR5SVhcbZjGzlSjIhaZ6oeblU8PnJzgvIwFZCpG +AUEniddtQ8v5cOYSwizV40JnvkYQ2/3b4bnlk2hzJZ+ntk9TqxePo70nPwVbRmxh +/h6hbfP89BgyNr0zA/SvDAFapIXcOnaJxUiG2zqaMQTl3hs1AzrFuxsZnmWjo4o3 +Y9V2KsbX9IcIVm4WFmj1VIZGa5t4wPM/FcSv48StHShroM9xlB5rKSpMBc6Fa1dz +vgH8tcJn1lyOnCzNNR/uFiiy6KsDCWRfK1UgLRbYa/cbI4AdAQKCAQEA3Y5cep9u +6i7LC7A94gxWYhBeYi1bTdzczqUJtW5ojTIrm5aS6ngRKym3Rn+j2sIsbFSvAM7D +yDLEWgwqlMHknLLyyGn+1f5HpFHLBhBJC9QD1qUA7luMbiB+N9L00MUpt0YCmAp4 +pNMxIN4fcUpcmj89jk7CN7tMr7keBik1Sko0IPjHwRKL/qzmY7oy+2qqJiUWGlCq +LOiNIAmbzh5eoRVodgEw+40f2vXFIeC3YxC5HX/GupXul4QQ22As/RTJKR4YcOlq +hKLK7NbIRWIrM/MkG67zjNOm9DkAwZKRP/Pdm8MRQS9m4/OYIOQ+dfD5CmqtpYjY +UCEgmV1wZ8FlQQKCAQEAzC0G9/LwAXsp2FXRD0H9FgEMnQzgDeem4XBstssMcup5 +AuNpXAW+SC1OSLbscAoSCyN5FlbEhrhalpyzmMpDMWaA98hjHsVe4/uXldpdtYIM +2Mdw64ejsUteQ9dMAQpEA7OXHguPD9bpD0jFdng8PRxD0FMb7OpHUsMkQkbRxpgR +/dQr40oukw35j0AO0jex8CPZk74W1FM84Mvr4YpXZR2fPawpq6DtkMuUWShfjp2L +DKfpi2fDqLg3ktphjwK9V137oUUqq5COwCMSZSYehEs/lCo8t2IVYnjG1Y8iTeIL +p/SXuimKjkZab9pIWzexRdeBCBIoREyMR2ka1B6tCwKCAQEAh5Y23EgzzZcjZZzO +0kawmsTfrf+J7+WpL8FqZqER1M8ORuJoOJuQDjUyVy+H7DmGjClrC/h9ckg3R9BI +Qs9CA++zipjrfrRlzZbMw4I14co3KyXBox1dnIvAEpPEhPsUI3fx4HAQEva4CyBE +Z8WVVEQ9fJiQXSD/tk54pFaVqUguDLuTK4IZMJtJR2DNEHXfeQabNzGEY4VR0Hhw +5DxLtxLt4cGq41cjVpGvuaLXOAYk83ud99QHtqv40PhTr/IHhWVImdkO+erSewya +dXO1cs5qm5J+tAFXHADCiLQUtemdBlTje1vyALCzufVUkEMPdaOBb1uG3crYM2Mu +hiChwQKCAQBGHNw8vv7t80ZeVLHSxH1NVhTnxWkXPUpQNhQO/VEMETp25HLkt9As +5unwDGjK3JpvmsHVCvy3qozpFWyS9jn2t9VnwTdQdMRuE5JtUixFzR/uPSaoqs5q +bpAzSbZ+0PCkOMKW/a8l6E3mAr5UPVuqPI+I9VEsnRLa98n2vG0C/muHN5nOlkXC +F9Vv/LZaN+fxp83qbydVyDOqmjwpGiGCxpZNTlaM6llKFK6foz6VMcJzGSIYzgwh +Xwkuc7F2s6EjZFGpgp/xhFWQ4zpI47stAM4Xah2pd5QSWZkl8t370rjf9x03P1am +HsClklawhEJq0mMry1hqg8LULuKYnuQbAoIBABvXtvqKhhwsdxLQwsKcWDAqPVJV +bFs6Evvz+MZyeQxkCQJXZa4q07/65C5bivu09MkLLs7K9Xe1jfjl43S/yVd/7Z2J +bAxJvC+TGcZXgtP43dCr3AfRGYt8x5CuXWRJNB/HZL0MIm3otFeYuoSOUinUZmIz +/+rOAinLgTTYEC5AQ0rAMd3nJyF9bCvJKK0KJjJIQenkH4J6N6Y004y2wUMV3qvQ +4hB0iIUSae77nNq908VMGd9mfVEk8sSEussv0rCUDhE2ScSTnJr2Vyn4SKZuzO+C +ZMZwR90M626GIX8Wmt0cathiqd3qs+XhbAZvJ74LVAw/eEBC4FSoJ0rx/0I= +-----END RSA PRIVATE KEY----- diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java new file mode 100644 index 0000000000..3d87c02c2e --- /dev/null +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import org.eclipse.paho.client.mqttv3.MqttMessage; + +public class MqttDocumentStore { + // should save the message in a document in the collection named as per the topic + static String[] collections; + + public void MqttDocumentStore() { + + } + + public void saveMessage(String topic, MqttMessage message) { + // TODO: implementation! + // insert message in Document + // and insert Document into Collection + + } + + public void createCollection(String topic) { + //MqlCreateCollection collection = new MqlCreateCollection(null, topic, null); + } +} \ No newline at end of file diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java index ecda5df655..d2aa6714b3 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java @@ -18,6 +18,9 @@ import com.google.common.collect.ImmutableList; + +import java.util.Arrays; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,7 +38,10 @@ import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; +import org.polypheny.db.information.InformationPage; +import org.polypheny.db.information.InformationTable; import org.polypheny.db.transaction.TransactionManager; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; @@ -55,10 +61,10 @@ public MqttInterfacePlugin( PluginWrapper wrapper ) { @Override public void start() { - // Add REST interface + // Add MQTT stream Map mqttSettings = new HashMap<>(); mqttSettings.put( "broker", "localhost" ); - mqttSettings.put( "brokerPort", "5555" ); + mqttSettings.put( "brokerPort", "1883" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttInterfaceServer.class, mqttSettings ); } @@ -90,7 +96,7 @@ public static class MqttInterfaceServer extends QueryInterface { private final String brokerPort; - private List topics; + private ArrayList topics = new ArrayList(); private MqttAsyncClient client; @@ -105,6 +111,7 @@ public MqttInterfaceServer( TransactionManager transactionManager, Authenticator monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); brokerPort = settings.get( "brokerPort" ); + //subscribe(settings.get("topics")); } @@ -202,14 +209,14 @@ public void onFailure(IMqttToken asyncActionToken, Throwable exception) { log.error( "An error occurred while disconnecting from the broker {}:{}. Please try again.", broker, brokerPort ); } - //monitoringPage.remove(); + monitoringPage.remove(); } public void subscribe(String topic){ //TODO: trigger from UI. - if (!topics.contains(topic)) { + if (topics.isEmpty() || !topics.contains(topic)) { IMqttActionListener subListener = new IMqttActionListener() { @Override public void onSuccess(IMqttToken asyncActionToken) { @@ -224,6 +231,7 @@ public void onFailure(IMqttToken asyncActionToken, Throwable exception) { }; try { IMqttToken subToken= client.subscribe(topic, 1, null, subListener); + subToken.waitForCompletion(); } catch (MqttException e) { log.error( "An error occurred while subscribing to {}. Please try again.", topic ); } @@ -298,12 +306,51 @@ public String getInterfaceType() { private class MonitoringPage { + private final InformationPage informationPage; + + private final InformationGroup informationGroupTopics; + + private InformationTable topicsTable; public MonitoringPage() { InformationManager im = InformationManager.getInstance(); + + informationPage = new InformationPage(uniqueName, INTERFACE_NAME).fullWidth().setLabel("Interfaces"); + informationGroupTopics = new InformationGroup(informationPage, "Subscribed Topics"); + + im.addPage( informationPage ); + im.addGroup( informationGroupTopics ); + + topicsTable = new InformationTable( + informationGroupTopics, + List.of("Topics") + ); + + im.registerInformation( topicsTable ); + informationGroupTopics.setRefreshFunction( this::update ); + } + public void update() { + if( topics.isEmpty() ) { + topicsTable.addRow("No topic subscriptions"); + } else { + topicsTable.reset(); + for( String topic: topics) { + topicsTable.addRow(topic); + } + } + + } + + public void remove() { + InformationManager im = InformationManager.getInstance(); + im.removeInformation( topicsTable ); + im.removeGroup( informationGroupTopics ); + im.removePage( informationPage ); + } + } } diff --git a/server-cert.pem b/server-cert.pem new file mode 100644 index 0000000000..8e8fed913b --- /dev/null +++ b/server-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFoDCCA4igAwIBAgIUWCL8APe2ccKyBSaFRiYC9GSN2p0wDQYJKoZIhvcNAQEL +BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES +MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs +MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM +cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh +cy5jaDAeFw0yMzA1MTIxNjE2MTBaFw0yNDA1MTExNjE2MTBaMBYxFDASBgNVBAMM +C3BvbHlwaGVueWRiMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAz5ww +3zbFsi5t/PExS/0rv2zSiAdp5f3eR9IB8uXe3oE6TriZXIDqu3Hm2yd09EDp7/Rq +7+2wt/ioYojutHNKzX8vIbN4WEv4h1dahaILhLlSoVkzqEgPn5EJ7ZSHM97EJbjy +zDnyGboQEc0uwW+We9Qurqf2lKh9QeagVON9lbRsWR+GuoPh7YYreCT4T4r7qBDf +95b1yAG79G5iutMpkKNkyDIKQL4nKWAq89MpC+VvBnIS1iZ40XiyG8EAgyLhS3ps +tE9QmCt4jf5BE5iOohK0edZK9GM/BR+Suz1N6WA8L8s644DjLmg8Q7ENtJMtr55Y +LIpqeh0LenhPA7OGnSOgslMoauXQAnU/LUP8mqx2bWDggzkNSanPf2w6VsSN79o5 +Bt2RTG0na0MFe1ibXUWsoui8UeITRvo6Srbx2MMy1rToNGKG+oO7ZfMghJuqkgy3 +lH0iiF/R/XeWo42TKqsbhOK3xIjFrUoC2OyghR0Ce6XjmVRPeyrJSyNayxmwpo1P +hJRzQMvz7TtB9YaCltecZXPtwc7SSFW/lUNIdFE/E56OxesJQZaDlSzZ/eYBfQIc +nkuDnlNJ1IpWLtZXZz2Ir7gSH44FyD7e0Qeau4bkxekCBLwDRubat2f5qKp8uE+6 +fk2FQkfa+cJw6rS4AM49CMXSoO/oWCxoQtXV/rUCAwEAAaM8MDowIwYDVR0RBBww +GoIMcG9seXBoZW55LWRihwQKCgoUhwR/AAABMBMGA1UdJQQMMAoGCCsGAQUFBwMB +MA0GCSqGSIb3DQEBCwUAA4ICAQAMFox5y6muoQn6avrdNVa1ILlC1XIRIJcwg7H+ +Zj5Lvg1VnVjQVMVP50GvBxjPSNo4UE0omd4DvAJYxNKTOAbsFSXal29Xild+yOx8 +9GXp+hfU3I8l1Iho20X1VqF/rpOvrn0XTcFvJ5MQNEvejRvXD8fOM2HJ27I+FGFN ++b0gCSX4cjtQSnrlOxSbcG8r0emI1EOu0yXnadlDUd0rVxdVHyvS4dFq3bdqEinD +HclKY271sew0gRPauI9gn/iyoQLG457ibFkDimmaKW4CxvbFXj2NnRd/VGUmUaX9 +38dEh97CPk03mY5Er9HTrilTwdnGBjMfBK3EMxwBzqO6IHZTe7uwuLHiBJ4UeMju +iWvFqEkzOXmuTpWX0/qVIUqLxwVJRAikNU2EyxcZV8xr8GCWL35n2dx/LNHaf390 +mthuHqZpGXMia5zhGr3/puzX9Kb1hWk1HMpaagayYYsuOHMcSoGkm/TO/xHcDQAT +FkLQBWENxMw3fCZ77CeiOj9HIZ+NCrvXPnqZsQq3xOJz8pJUH+O/c6xVXceXpatk +dZuwxHREv5nE+Z6Z8GTMLFNhHUCJ2OfyglPyJXQEo5aDGlf96RdB/qSX7hzCfPNi +h8zCxYn6AuwVOZqxh26SxXIRgA2kUS5UnqScLvIE8a5J1myS45J5DluV9wDGZHA6 +TAf/uQ== +-----END CERTIFICATE----- diff --git a/server-key.pem b/server-key.pem new file mode 100644 index 0000000000..75449e9146 --- /dev/null +++ b/server-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJJwIBAAKCAgEAz5ww3zbFsi5t/PExS/0rv2zSiAdp5f3eR9IB8uXe3oE6TriZ +XIDqu3Hm2yd09EDp7/Rq7+2wt/ioYojutHNKzX8vIbN4WEv4h1dahaILhLlSoVkz +qEgPn5EJ7ZSHM97EJbjyzDnyGboQEc0uwW+We9Qurqf2lKh9QeagVON9lbRsWR+G +uoPh7YYreCT4T4r7qBDf95b1yAG79G5iutMpkKNkyDIKQL4nKWAq89MpC+VvBnIS +1iZ40XiyG8EAgyLhS3pstE9QmCt4jf5BE5iOohK0edZK9GM/BR+Suz1N6WA8L8s6 +44DjLmg8Q7ENtJMtr55YLIpqeh0LenhPA7OGnSOgslMoauXQAnU/LUP8mqx2bWDg +gzkNSanPf2w6VsSN79o5Bt2RTG0na0MFe1ibXUWsoui8UeITRvo6Srbx2MMy1rTo +NGKG+oO7ZfMghJuqkgy3lH0iiF/R/XeWo42TKqsbhOK3xIjFrUoC2OyghR0Ce6Xj +mVRPeyrJSyNayxmwpo1PhJRzQMvz7TtB9YaCltecZXPtwc7SSFW/lUNIdFE/E56O +xesJQZaDlSzZ/eYBfQIcnkuDnlNJ1IpWLtZXZz2Ir7gSH44FyD7e0Qeau4bkxekC +BLwDRubat2f5qKp8uE+6fk2FQkfa+cJw6rS4AM49CMXSoO/oWCxoQtXV/rUCAwEA +AQKCAgBff1cztfwmRBkv2rp0W9Z9zU7iZ5acxrQykTV4R6jG7OanQ164IEzZQEGS +2YmAuUNirylpiApb9bNu79Hti4MDF9fURru60edWSiZHhgyKVrzdnyyV4d/d5FeO +w12VzIipWXPF4Xmtv9qYdatmKzArcaSvjm9sXjyPzcg+hA5GiKG2NTxCG1wXaDoi +n87gt+uemXSMbRJNmA6fFw+vJGbgM2iudeYlddKJw8S+N4mQAU+axWpIu5WjtUMo +ZRUZPTgpAf1yy3ElLgncNvI0U154yuXdjWcZF3Xro2RZYI+wJCZLTc6ey7C0rhna +gVzujSzctZeVlW35Edcv1Jtz9maamPyxA9jVttLXdQkpw3yAP5lvYyacDND6s5Ob +QmJYs/ARhHDUCpWhh/PXUN1oIOqFxlJcJ9nD6HarDftuyZ9R6LkKlD9I8tIO3td+ +gvbKtdb+2BMf3559vGwHkl31bq9/Cb71n/3W+BlHkVoXfvvkMwh+2FX7weSw08i6 +84VQGwAQDi2xV1DK7leB9oHhLQh+bRw6+VeSpJe59twB/f+SBs6+WgoAnuE77JJ/ +xh/MWDCGQIzMpishmAiwK8Vt3HGXk2l3urRu2xsUHtA+xGqBaMsn76TZaCptxfhc +b8E6Dd+90bBIyLtleFjiHtrwMRgWNHcwa6xFONbmX0I4Lyv+WQKCAQEA+ckgD9TE +fnXgDvYUbpd987JTYePpYP+rE3EtnZLvwmG7GG+z57u3s/HJ2ltW/+tpMvNH5W5M +/i/L+1gRrs9R46kHfNxlWORyS5/U9LQRGru1nqyG2vCP1PRycizQ1rbx77GJraoG +Kgrg+cOshPOaISmtU2D0fpRunQEyE2E4guO3GUDkgxlc7WYVNMCE1O74aurPLVUI +6TSL3DNguaNzRwuhDBpTTS8+YDSP8JBTp4xIo/v8lnyxps1J2MF+OwKI+bTyc6AG +KRwqKwEjBZlPOam5WyhS5+gI4IGJ0OVr3ReKBy+mV0Y9U0Qtr4bM3l24EeoYJm9i +u725nyqjoB0pzwKCAQEA1MZzkX0LNOv9nPxbQKWRmtfHEztQDaVgLvM+yQTBFtpH +o41MaNGfaPLJo1I9xM1QGjIkSeQXhQuED0CLog9fc/QZ8Aj6r4ayq+smhGZBOcFs +StXpJUZSvJ0SYV5pW2S5ddj1zFoPxmGIcKNfNkYwRUv+F11uXkLl5OnSaCL7mTpS +Bsi9ebvVtkgAxRn+SBd8v6evQ0wKIUt6Wd9rbhD0SBnkzMZj1UEzWLKcm9OSOotM +CMYdGtBoaM2FEWzMLEK/AX7k/dMmSfn6y1L5kGfa6FLnxuSbP4gcPX6KGeAzOyFz +f4ByA+ZYGCaLvZygaP2BpPWJjmoujVTdgpYywU/kOwKCAQA0A4Swpz6vkr4wf1+r +09H/gB/K35nGgBbQup2kFWfl01IReZk95SIQz4WX+bM5IX1Lhctv5DMWLz/LvAAa +pcNIo/cSMhbMlu2R/9tphWGWboYYX5bJWpSo5Ko8SkbGn1vr7M1do20bLh7BA9UV +7mAnDbWxChx/i0spWIlj7bfF0cNb995dyldGfB5oWLqQWJ7ULMctimqLnUyOcwhu +P7IOlkZX7ddw2CimAewDTRqOZjDuLNfhQl//+vmosEon4ZRCY0KlfyopP1ssj1UK +om0T9drKN7Bj6DaJAZCANnXqmS895dc+jzrP7v+QcIKSv3NID7Ytsr4dmBliIo/m +FIyTAoIBADONrdy3Zmq5LybFQjrkouXYmESBn4De6/IUP8cm3r3xv1/SUYwoFi6w +fMiUmaDAekMOmRpKKKS0yJeFdQ+cF+2ZXFt7YXn78fiTri5wNCeRRPLQ539/jXZj +vFtZK7/YJNN38P5FaAMSCKP6AYy97t4rbnIs0hwGq3sOaEuRBlL2/X7lTntbVomm +4oDfLmw7PJ+XOnDc0KuGhSuXZYkylG2P0I8JFpFKDe3UwPtEhO6oZ5DMp6qiilmy +SiaRrNqghXUQUN1rNqC/79Mp1Iv3jeOyirP3GeKH0QfZTA6+8srF7R0cewOU4Q1g +8YTMmVU27x2P3HdkJMlboZwNVljeSZcCggEAe22CCvR/fQ0jlOeAoGUX+KO8AXSD +PQwVsB4IxtHBXgeIdrr0qNec6OIAJdmn1YnD0jA7dlQ2Kl08P7wR9/SJgCZZb8lJ +bW/4QziQIf8gDRJNDT/7YkQbSbdtdCsQN4DqVGHLoCgGwX/YuT4zzWywo5q95AM7 +Yyo7n6ehaQJOsG+2YoTswddyqhVN6r7pbJY3K1/fYLcIaHtDkbyX1QyMlnE1vIFg +MDYyLLbdQEORgTpEmA80HOjqujXlqwX3wQC9KJb16cBRfvz4kVh9smo8aFhVZj51 +wImXjw60CdkA6b3oyvmFhKETJnJl8AlzyfODbxQ59FC9j39TTSYhKwrnmA== +-----END RSA PRIVATE KEY----- From 247892ae1f8514b06b04f5a50be362d002d140e9 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 17 Jun 2023 16:51:34 +0200 Subject: [PATCH 007/114] deleted security files accidently pushed them with the last commit --- ca-key.pem | 54 ------------------------------------------------- ca.pem | 37 --------------------------------- ca.srl | 1 - cert.pem | 32 ----------------------------- key.pem | 51 ---------------------------------------------- server-cert.pem | 33 ------------------------------ server-key.pem | 51 ---------------------------------------------- 7 files changed, 259 deletions(-) delete mode 100644 ca-key.pem delete mode 100644 ca.pem delete mode 100644 ca.srl delete mode 100644 cert.pem delete mode 100644 key.pem delete mode 100644 server-cert.pem delete mode 100644 server-key.pem diff --git a/ca-key.pem b/ca-key.pem deleted file mode 100644 index c7db71eed3..0000000000 --- a/ca-key.pem +++ /dev/null @@ -1,54 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,DFDC177D1AF280A6464E4CE7508715C2 - -E2AnAgLZLPKO86OS68f3azVfyU0j1nkMaNvCJA+fTxJhl24gairOevgwRwScEVVJ -c3gCT2d8wrnwSJiuhCg0MIas5Wl0xD+JNI3ED0x0qd/BxccERb3Ie1+aC0n8VG0N -E1SMJ5BNhp2T/eAHSRzVV3y5yv7mnZ24sz0sjkhTVvc6X8R4kMxRVfVXUItfmPhv -CWrl4eC2iwizGXZq6f32wqhvJN8MIzltKxuwlUFTZ2RFdxSzLr3X4bJzc8b94LtN -HxnJBFWPrSxgIDVi1Df7WyM6XKURgjBHuedVwV4PYUdsRc/mx+IQAZY/kwHcsPSa -AlmlitOOP6MM4CtPDj2YB+Y5XmXwXawemAIsG+TNTbFVbYZ6YaZSjoM943D7ufvu -7zwxBkXqh+BC8dMLYtYcvkUI5uYhXhPAFmBkzfnsHccLO7qi2MDNWWjn0k52ufmN -mb4ZHAcWS66RrPtqm1fr6I+yBbtWsF8F7McNiuMtQ4qQTg8BFM1OketnhK50IQLr -BSsGEftd7DbBXyqRMKTJ2hcnrZZ/Lbt258KOUDKcdnM27xSYg5mZX8JuynV778hb -YhcGRAyO8K6Nc5X8iFsxL6xbLlL0aJaQE6RHRnr6yLDNt8R4K2VfIwh5/snzGb8d -4kMWVecykqACqf32EZPgQy4Rt/Qq9pF6aI9pS4DJ9yHtP/UxrLZ40ueiq8qbjpDJ -TJoLMmHKx4nPcjD+a0/uhvmzYft6EOZeLVpe8NlIAf8cEsw3nWfE1LaiVpAUxZaT -qSHbqQNSEeo8hwU5UGpCC1uLj/x/olfyN2DNZJZOa8J8x3+SDWH7u3MGiaw+mTE0 -hIwt/UOD83Lgygp4RDOYmmFKM33PhBm6V8Ejuwa6JKqVz/WjUfJQ/4XP8IjlOcTd -4P41OWFPnoeXULBgBF4TRd/7L/k9UPScXSitfZb3LFnkv8edH3a0jcbM+FRjZymY -LKvjqiBUbK0tk+SNqjlHSjBWy97V13qdVTgrkq5wCJ7FTtcOvdR8XoIy67UyLFva -C9kg8xCh8MNLezKCm2+tfeDVD0HXhSfaw+8yiNbg4k8r2dIQDRf91I90MH/BZlE4 -fOwS9DrMxJxNzjUuM2PWnllr7vubfn9n4jMw1IX2giTVlYj6CFCP5bsGGeeawlk5 -uFaYakTdfiwDW3YfWI2g/n3I+us1BFp2RivzJYehXFhgDfDDASOROU/QSB6pkEn4 -GylWbvoP9qTS+V76rS2uhGsfSBJhu9u4TYiplpBjtvgiCXsjuTb7m0nNv0tyGiDH -BKRdU2JqAINjHGl9H8qjT7KKrkDF+uib7qBpXoQN+Q++CwitzoPwKUFS+6d3R2Bl -eHfaGYvAdYV3QrmKCnUGZdmNVTYARxTFdiKEEHoSzrOrj3puB13BWSxcB7KjaMQl -/wkNF9DCLpRzPMPRkCgWnFCncDMoeA5z8AmwBu1b4z/pAP+RItc9ClmJv+xgl27v -v1FWNxHFMBktzDXUPg3sYXUVQ1p89qzkM8001t4fU7HsDuczNt0t6TEx9xlypU4w -FMI6htsQofRkMqAvquLET98hMS6AC1p309V2gxpE9bgvrZWtAd2jt4JVqyl9EN98 -UwmJQL08k9c2bApLZSX0af895xTH/PGdBLFzjOgnBBrQ6F4D5zUls6FZIhfnFKXM -ATQmbZriaohQAGlblzV1UlWa/x0xpjeyvdv9eCj5G06SteLm7w+gJyU0AZgcqYcx -NAcCa8x+auYQOMQlDk901yu9/iKFntKchL5EwUIp8+vTGqWAX/8OS44ULIfqoO5/ -ZpCL1Bc0XbXWKrCO1NpZj16lCYuz5Noajx+kZZsWbKubCy6BU1Fu+JUUYGTkZSPF -AWgPNLNKHfbX+0PkdXv8zmw7OV21JqOSy6eEqeQ4zY/+k6irTpPz+DBnT7AQdvZ5 -ZBnFU3AdjLwtKe1xjgteBygYT9d0SAkX5N8nov7tnt/TDpVPcJU4abPH/PRfAXCe -wNniIfK8nQWfjjHeZmS4+hX+oqWTnHynKECzmnIDm0oqY2h4Tgg8DHwwfCC6Oot6 -0sAvcdSF2csG5E/yr3K3xjQKh73Nf+NdggTJFypba/bEwZgdSoaJM63agh62GNuC -d8KphnJy9OQI3fwp5ftcpjJgHVuBaK06bMR6gAMWCYH3iomRqU9cLUt8/X4DI/6d -M8BA1ZBiJ9s922GSzQelWaONcqOCb0k20IaAI1rwjCJqrWCl3p5z1E/3X1gKhAPQ -5e2dQJo5qctDLi8nHin8IZf3GD9iannzzUxBAQhKP7HSDY5lNcP6d3qpHilKpccq -kCOMju1JDDaUfcuzbkDIEJ9E/eDPcgXX+ukC6Z++Osy398NE4RYagKl5RXZtuLXL -s32Q281H6KkOCG+V5OccGqi5EO6AzP+GAE6P61jDqWdynLzUgRqX4Sgxr6K4zVrt -TnAFG1X9NwXqLrfMvLD/lMR5aAkY41gUwBT0AF+bWGHMD7j5eNNI8cZLBOk1cKa5 -4bfhE/mkJ37vPxQa3Vx+03fyr3T4k1iUlqiPjnHZ7CaJV1BBDlWJt87zDUlHJyd0 -1SUMQ/6sFTB0oXNN3uW1V98l/tT+7iJ7orGCFwrdRxgbXsc+sdOJtEoUfD01vIns -txGxEf4bQlcZi2pQkJe8Ov4Djzsjyf8fYZ8bRLhkRaiVMvUPvJXv1AvfDFS7Y2Vh -BdWJtJMkRv10olxcRzzL1uoH562QpYuUjoscblHMBP1SWVODy5ou/+c2xBefg59N -71On/7DPC3OO+TAbirNR3nEsAVwmlSsygB/7l0xI8Q6tcFwJx7NgIcvy9mW/fTW3 -YbWC98QEGEd4Zz60+itDnAGa3vPNKR1JbD5Gez6WpJqRdY4SroATqEjiX9c03tJH -YRxW126kNmQkoTunXAwXuhAUSv1CpFQwep6Kjk0ySEfatxIbddujSveIgzIlIdIn -8XVJqJgerJ//bw/A+3ZNSn8UZYwM2hcSF2GFPIXj/cNERMVMDoffvnREvpD7yAJQ -LOJ0+uwyAiULSI8WkzLe1xGiWHaWUiOuILaIQetRsgO0rujT9qRJi9pv2ZuLY9Um -/lzoCyI/8Gyn2KF9k1O2sDcPEEQRPm1BM48O0G5eiQshREWvEyWyvIbHfIBHuCUt ------END RSA PRIVATE KEY----- diff --git a/ca.pem b/ca.pem deleted file mode 100644 index cfc3f92fa3..0000000000 --- a/ca.pem +++ /dev/null @@ -1,37 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIGYTCCBEmgAwIBAgIUeA1gR7WY00gfgt2m7y+u5ZbjlEgwDQYJKoZIhvcNAQEL -BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES -MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs -MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM -cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh -cy5jaDAeFw0yMzA1MTIxNjA0MjhaFw0yNDA1MTExNjA0MjhaMIG/MQswCQYDVQQG -EwJDSDEaMBgGA1UECAwRS2FudG9uIEJhc2VsLUxhbmQxEjAQBgNVBAcMCUFsbHNj -aHdpbDEcMBoGA1UECgwTVW5pdmVyc2l0eSBvZiBCYXNlbDEjMCEGA1UECwwaRGVw -YXJ0bWVudCBDb211dGVyIFNjaWVuY2UxFTATBgNVBAMMDHBvbHlwaGVueS1kYjEm -MCQGCSqGSIb3DQEJARYXbi5zZWx2YW5Ac3R1ZC51bmliYXMuY2gwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQDCrtrmEPaXKaKrtTklhCv/5ayoHXTdiY8j -DxubtXSgk0v+waeGNYrvmLTSX25Jvh7FEFVvdT8+A9aB9sh0SmDqLMsjDI0v/lB+ -o6aN1BhZaOxPef9lCzGdyxm3Hm5Ww76Ra1rstjRUmw9RXn5S4aFsYTwLt7s7Qds0 -QuVT9Zi1HcAcgqwRBYysaBb9I7p59LursYCcx7ZIhiuWESJNguzV00r9UGZTS56R -qOjLizJcGEuuL38iUNsNHVb0IWbW/nGgYzsKZkZMvEBPoA0Zi4x0b1EyTGrwkOQ7 -AsBho9uVkbrQwydfjrX8eoaQxM0K9m2IEZi6Sb3B1b3wW16dw1mS/zxjGHDtyogA -RCM2Ru2GwC4h8WoT2VrKgReDn0384pOjVM2m84y19S5U/6iFvtx+DqvU7T/FEaiD -8l8oGitlc4Uj2/ls1q8OxZbck1RlCNn75UbOuw4VyjNgqBvmdrdsj9Hb5ofyFm5+ -5SLtVPLKRKm9GXKmp9uo7CTx6UtX6p3LKvllW8czu9GQZnGvCn8sRnuIInhheeUp -ymUrBf64BTvosaZusIIsOziqlSrEp2YcI/8iHV+++p9SsChjU0FPALiTrZJ9SAz/ -OWqstBW0QskimW7YBYs8FPWYGTanDirGdSLbc5o2jtn3x3DFkA8BGJFs12EvcW0g -a/iQgW34uQIDAQABo1MwUTAdBgNVHQ4EFgQUX6ncnORVSzHkGNWTjWNGP3YG27sw -HwYDVR0jBBgwFoAUX6ncnORVSzHkGNWTjWNGP3YG27swDwYDVR0TAQH/BAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOCAgEAtM7fO11MtMk+XMf3m2nO8pgRwB1B2LyysWRW -iyzCVWvVYQJIiVU3BfLZBlCjDqcwFzSOmREX4qSfxDsR+vzEeBpxLce7DoRvJrT1 -8ab7+wUWne4/GrtJU1nZT6SuVgWpgw1qISJ2cHkBvJoR0aGu6q63nzzg1KVOe6d8 -DLShMIRDK5uRGBtKJWnyre4JSDo0PdXkflb85XkhQ9rc72LsVQ6S/t5A2kBQptk0 -Uh7Adbt1HVr0LXEujDQxMkvclLeUbgHLBCmAeD1xZjEp7X+zT+iykG1Bx0CnpVNV -/w76XsiVvLqEvnene5O3qZNmKRXhMe2X3mGHDkgtFMbRIsX0AyMiVjjnJQyJZ9Ka -8c7Nah89ldWwyy6h+A+s7OLCNvH0F1kTgwLGGTtXoF8BMqrtQ4/HRLZTNWwDBG9S -wBW2uIM/j9DtI/AxiglwtJcJZe0/Yg+B9yh9+lVij+yVIUonqe9Db7tb3QjR1PF8 -y1kaG5UbHmLEa7hiybd0J0ZhbfAlq2l8GbvqCm7GhbIljKrwnw4YKwgDc3dwTn9T -emQs12FVXSMPtsLyhsCXRKR6au5N1eKavntrn6R/njyHbB1KAAFI1UM8/xiXBq0t -lAxr7czdmMYmaUjQGuxA3czlmOZN4dPQZ0LGCyM/SNcYXAPm3Kin8kYW8RzmnNhm -S0JHkyk= ------END CERTIFICATE----- diff --git a/ca.srl b/ca.srl deleted file mode 100644 index d2de10eb43..0000000000 --- a/ca.srl +++ /dev/null @@ -1 +0,0 @@ -5822FC00F7B671C2B2052685462602F4648DDA9E diff --git a/cert.pem b/cert.pem deleted file mode 100644 index e2e4e04f6c..0000000000 --- a/cert.pem +++ /dev/null @@ -1,32 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFdjCCA16gAwIBAgIUWCL8APe2ccKyBSaFRiYC9GSN2p4wDQYJKoZIhvcNAQEL -BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES -MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs -MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM -cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh -cy5jaDAeFw0yMzA1MTIxNjU0MDJaFw0yNDA1MTExNjU0MDJaMBExDzANBgNVBAMM -BmNsaWVudDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALC0ab/lYM1T -pS5PgCiRHpWGib1Qy6ohIv4Cg9qUN6ghNmiatIA4O5JzyvJhK2HX3ojULNwjmost -UjhO8jjtqUULEeI8upFeeQF8ptGXJrCBeSp9ZwvB/ROheLZkZrkUFd8vTqXa2WBh -qE4SQ6xCUl5mnVx7/9mA9sS+rdvKLV3w6Hd6KEp09xeyqOBE5+tAh6JLe+QOJe75 -kHO4xqMnq0pVJJ/2fphEThCIFgTD8yMDvXaeWRLZCYhJ7F4dHCcFtrl9GPlnd7II -N0PXpEJb46QUd2Dhvg29jcsCVIo1mHWppi+MJyIgZ/2f1YrKquRTRYOaMAuc/grK -RY4Vv9iW5orfq4dZiOxrF1p4Y4NGJLuGtPOwIHoFFJZHRk0fa3AoGEvaY68/8tVm -HL8jEcnsKTzS7oQU5S7tOXOBO8A5i0YVcKaVM3YkncKfRzlLKPs6qYiSwantFYY/ -aJRABbHv0S/nCHbaaQUZV8L/SkFAsaBOz9vyW4fd25bbaohiPTR8TwLHve84hpyR -4esOfv4TEbZYy5/jdwKQHg4RnKXjWoJTbsp98CClOR+3hKyZn5SVpouak+4BMrPh -bxWjcd+nCJYBPFSOFaqDdqLKbF2kEXsYanuXXRuEuKILokXcFKNm7ra28DNKzsXj -fIn0d+1RQkYaXnX3DJsdFZEHPA/YWkbLAgMBAAGjFzAVMBMGA1UdJQQMMAoGCCsG -AQUFBwMCMA0GCSqGSIb3DQEBCwUAA4ICAQBEGt7ipiGBeHhNn8DV4SaY4uIYjti3 -B9PuSCRL6uaGpeQ8rESTBzdcpr0bHjCQvLh/ceY/w8PAGEihSzbOtfpBq9V/4pi3 -D21yiT7Z62DMl9G9bK1KD5yzsCK5llap3cBSqAVO+6rH72k3Awu/EwG3kCG/s8V4 -7nC5snml+hDos14G+NXz17EpybeaM3Ylmi2O5SJHUpByfKcX4sFOUgpHqg4miFcr -1RdSnoW2witIi8FI/LCVCXmsWU4I1KXwOFKpXFVD/dmyyohbU4gNn0KAY9lapFoF -onR0RpbmXMgS4fjnHjgSNYsofSrTW3mhLvYGy+iqAoDDvJXQMAbIKMjAgHeThnB2 -C0omXt1B2bUYB45OOFDpm3pO8b87FW8S94vnVaQ05UmINDYVZ+V/uWhXArLEaA/c -VTnZd4yJklrXBHBUv7WNSuDl3ouX5t5PfF61JX/VNu1U35Pbe8/6PFT9KNisRp45 -9gcASkJHa6MA/avIV3bcwFjxIgkExEyGwxgQ3BjCd9XkaP2HA8kEaW1jq70nLx4+ -/P3CLQOVhe0a79goYYGIRsdZrQJLIUSXwdsuBTkk1l+5UkFmfd9hBdpJfMS53hDg -vEqbbQTHlPNaLkmyVfgH0hj/YeN7b+5pyD9ioIYJOBaJe5hDstQM4Dw8e/EX6q6B -vA7FeU7gmG91ww== ------END CERTIFICATE----- diff --git a/key.pem b/key.pem deleted file mode 100644 index 52463195f7..0000000000 --- a/key.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAsLRpv+VgzVOlLk+AKJEelYaJvVDLqiEi/gKD2pQ3qCE2aJq0 -gDg7knPK8mErYdfeiNQs3COaiy1SOE7yOO2pRQsR4jy6kV55AXym0ZcmsIF5Kn1n -C8H9E6F4tmRmuRQV3y9OpdrZYGGoThJDrEJSXmadXHv/2YD2xL6t28otXfDod3oo -SnT3F7Ko4ETn60CHokt75A4l7vmQc7jGoyerSlUkn/Z+mEROEIgWBMPzIwO9dp5Z -EtkJiEnsXh0cJwW2uX0Y+Wd3sgg3Q9ekQlvjpBR3YOG+Db2NywJUijWYdammL4wn -IiBn/Z/Visqq5FNFg5owC5z+CspFjhW/2Jbmit+rh1mI7GsXWnhjg0Yku4a087Ag -egUUlkdGTR9rcCgYS9pjrz/y1WYcvyMRyewpPNLuhBTlLu05c4E7wDmLRhVwppUz -diSdwp9HOUso+zqpiJLBqe0Vhj9olEAFse/RL+cIdtppBRlXwv9KQUCxoE7P2/Jb -h93blttqiGI9NHxPAse97ziGnJHh6w5+/hMRtljLn+N3ApAeDhGcpeNaglNuyn3w -IKU5H7eErJmflJWmi5qT7gEys+FvFaNx36cIlgE8VI4VqoN2ospsXaQRexhqe5dd -G4S4oguiRdwUo2butrbwM0rOxeN8ifR37VFCRhpedfcMmx0VkQc8D9haRssCAwEA -AQKCAgBZJ427flXrAuryhLYawa12r9zLIfrAKdvn4Qj5vZKfR7KV8IhlsPmNmNKq -nBAxW6Mx0iuYHnaZBhLXyi04P+EJrt4lEZLoSYwwDd51HRguMgUaRQ86xXOU9gFb -tSiG9PVcGBhMYjEPCfbK/Tgbr5Kzgn6e7QlXilP3TGWX4Vgmb6g6r+2oM39/+2pt -TRbegT5tyjwGAoVtH4mceXvvUdKAvJiwBMQ2uy/P83l4ig0ge4CaWqaCdfLzSqjx -iyprHdzllZKGM4gW0UJg/mon+QjP/muE/CDR+fOSm+hfm08C/seUcdhExajyNDSf -xEWY0dLe5WqcUWdztj6Fw7953hkO3aH920hGeCaP+91zFmLfOZOVIfvpXa/C0XmY -uezb1X6JEBeHb4l1qNnOGiWf3CR5SVhcbZjGzlSjIhaZ6oeblU8PnJzgvIwFZCpG -AUEniddtQ8v5cOYSwizV40JnvkYQ2/3b4bnlk2hzJZ+ntk9TqxePo70nPwVbRmxh -/h6hbfP89BgyNr0zA/SvDAFapIXcOnaJxUiG2zqaMQTl3hs1AzrFuxsZnmWjo4o3 -Y9V2KsbX9IcIVm4WFmj1VIZGa5t4wPM/FcSv48StHShroM9xlB5rKSpMBc6Fa1dz -vgH8tcJn1lyOnCzNNR/uFiiy6KsDCWRfK1UgLRbYa/cbI4AdAQKCAQEA3Y5cep9u -6i7LC7A94gxWYhBeYi1bTdzczqUJtW5ojTIrm5aS6ngRKym3Rn+j2sIsbFSvAM7D -yDLEWgwqlMHknLLyyGn+1f5HpFHLBhBJC9QD1qUA7luMbiB+N9L00MUpt0YCmAp4 -pNMxIN4fcUpcmj89jk7CN7tMr7keBik1Sko0IPjHwRKL/qzmY7oy+2qqJiUWGlCq -LOiNIAmbzh5eoRVodgEw+40f2vXFIeC3YxC5HX/GupXul4QQ22As/RTJKR4YcOlq -hKLK7NbIRWIrM/MkG67zjNOm9DkAwZKRP/Pdm8MRQS9m4/OYIOQ+dfD5CmqtpYjY -UCEgmV1wZ8FlQQKCAQEAzC0G9/LwAXsp2FXRD0H9FgEMnQzgDeem4XBstssMcup5 -AuNpXAW+SC1OSLbscAoSCyN5FlbEhrhalpyzmMpDMWaA98hjHsVe4/uXldpdtYIM -2Mdw64ejsUteQ9dMAQpEA7OXHguPD9bpD0jFdng8PRxD0FMb7OpHUsMkQkbRxpgR -/dQr40oukw35j0AO0jex8CPZk74W1FM84Mvr4YpXZR2fPawpq6DtkMuUWShfjp2L -DKfpi2fDqLg3ktphjwK9V137oUUqq5COwCMSZSYehEs/lCo8t2IVYnjG1Y8iTeIL -p/SXuimKjkZab9pIWzexRdeBCBIoREyMR2ka1B6tCwKCAQEAh5Y23EgzzZcjZZzO -0kawmsTfrf+J7+WpL8FqZqER1M8ORuJoOJuQDjUyVy+H7DmGjClrC/h9ckg3R9BI -Qs9CA++zipjrfrRlzZbMw4I14co3KyXBox1dnIvAEpPEhPsUI3fx4HAQEva4CyBE -Z8WVVEQ9fJiQXSD/tk54pFaVqUguDLuTK4IZMJtJR2DNEHXfeQabNzGEY4VR0Hhw -5DxLtxLt4cGq41cjVpGvuaLXOAYk83ud99QHtqv40PhTr/IHhWVImdkO+erSewya -dXO1cs5qm5J+tAFXHADCiLQUtemdBlTje1vyALCzufVUkEMPdaOBb1uG3crYM2Mu -hiChwQKCAQBGHNw8vv7t80ZeVLHSxH1NVhTnxWkXPUpQNhQO/VEMETp25HLkt9As -5unwDGjK3JpvmsHVCvy3qozpFWyS9jn2t9VnwTdQdMRuE5JtUixFzR/uPSaoqs5q -bpAzSbZ+0PCkOMKW/a8l6E3mAr5UPVuqPI+I9VEsnRLa98n2vG0C/muHN5nOlkXC -F9Vv/LZaN+fxp83qbydVyDOqmjwpGiGCxpZNTlaM6llKFK6foz6VMcJzGSIYzgwh -Xwkuc7F2s6EjZFGpgp/xhFWQ4zpI47stAM4Xah2pd5QSWZkl8t370rjf9x03P1am -HsClklawhEJq0mMry1hqg8LULuKYnuQbAoIBABvXtvqKhhwsdxLQwsKcWDAqPVJV -bFs6Evvz+MZyeQxkCQJXZa4q07/65C5bivu09MkLLs7K9Xe1jfjl43S/yVd/7Z2J -bAxJvC+TGcZXgtP43dCr3AfRGYt8x5CuXWRJNB/HZL0MIm3otFeYuoSOUinUZmIz -/+rOAinLgTTYEC5AQ0rAMd3nJyF9bCvJKK0KJjJIQenkH4J6N6Y004y2wUMV3qvQ -4hB0iIUSae77nNq908VMGd9mfVEk8sSEussv0rCUDhE2ScSTnJr2Vyn4SKZuzO+C -ZMZwR90M626GIX8Wmt0cathiqd3qs+XhbAZvJ74LVAw/eEBC4FSoJ0rx/0I= ------END RSA PRIVATE KEY----- diff --git a/server-cert.pem b/server-cert.pem deleted file mode 100644 index 8e8fed913b..0000000000 --- a/server-cert.pem +++ /dev/null @@ -1,33 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFoDCCA4igAwIBAgIUWCL8APe2ccKyBSaFRiYC9GSN2p0wDQYJKoZIhvcNAQEL -BQAwgb8xCzAJBgNVBAYTAkNIMRowGAYDVQQIDBFLYW50b24gQmFzZWwtTGFuZDES -MBAGA1UEBwwJQWxsc2Nod2lsMRwwGgYDVQQKDBNVbml2ZXJzaXR5IG9mIEJhc2Vs -MSMwIQYDVQQLDBpEZXBhcnRtZW50IENvbXV0ZXIgU2NpZW5jZTEVMBMGA1UEAwwM -cG9seXBoZW55LWRiMSYwJAYJKoZIhvcNAQkBFhduLnNlbHZhbkBzdHVkLnVuaWJh -cy5jaDAeFw0yMzA1MTIxNjE2MTBaFw0yNDA1MTExNjE2MTBaMBYxFDASBgNVBAMM -C3BvbHlwaGVueWRiMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAz5ww -3zbFsi5t/PExS/0rv2zSiAdp5f3eR9IB8uXe3oE6TriZXIDqu3Hm2yd09EDp7/Rq -7+2wt/ioYojutHNKzX8vIbN4WEv4h1dahaILhLlSoVkzqEgPn5EJ7ZSHM97EJbjy -zDnyGboQEc0uwW+We9Qurqf2lKh9QeagVON9lbRsWR+GuoPh7YYreCT4T4r7qBDf -95b1yAG79G5iutMpkKNkyDIKQL4nKWAq89MpC+VvBnIS1iZ40XiyG8EAgyLhS3ps -tE9QmCt4jf5BE5iOohK0edZK9GM/BR+Suz1N6WA8L8s644DjLmg8Q7ENtJMtr55Y -LIpqeh0LenhPA7OGnSOgslMoauXQAnU/LUP8mqx2bWDggzkNSanPf2w6VsSN79o5 -Bt2RTG0na0MFe1ibXUWsoui8UeITRvo6Srbx2MMy1rToNGKG+oO7ZfMghJuqkgy3 -lH0iiF/R/XeWo42TKqsbhOK3xIjFrUoC2OyghR0Ce6XjmVRPeyrJSyNayxmwpo1P -hJRzQMvz7TtB9YaCltecZXPtwc7SSFW/lUNIdFE/E56OxesJQZaDlSzZ/eYBfQIc -nkuDnlNJ1IpWLtZXZz2Ir7gSH44FyD7e0Qeau4bkxekCBLwDRubat2f5qKp8uE+6 -fk2FQkfa+cJw6rS4AM49CMXSoO/oWCxoQtXV/rUCAwEAAaM8MDowIwYDVR0RBBww -GoIMcG9seXBoZW55LWRihwQKCgoUhwR/AAABMBMGA1UdJQQMMAoGCCsGAQUFBwMB -MA0GCSqGSIb3DQEBCwUAA4ICAQAMFox5y6muoQn6avrdNVa1ILlC1XIRIJcwg7H+ -Zj5Lvg1VnVjQVMVP50GvBxjPSNo4UE0omd4DvAJYxNKTOAbsFSXal29Xild+yOx8 -9GXp+hfU3I8l1Iho20X1VqF/rpOvrn0XTcFvJ5MQNEvejRvXD8fOM2HJ27I+FGFN -+b0gCSX4cjtQSnrlOxSbcG8r0emI1EOu0yXnadlDUd0rVxdVHyvS4dFq3bdqEinD -HclKY271sew0gRPauI9gn/iyoQLG457ibFkDimmaKW4CxvbFXj2NnRd/VGUmUaX9 -38dEh97CPk03mY5Er9HTrilTwdnGBjMfBK3EMxwBzqO6IHZTe7uwuLHiBJ4UeMju -iWvFqEkzOXmuTpWX0/qVIUqLxwVJRAikNU2EyxcZV8xr8GCWL35n2dx/LNHaf390 -mthuHqZpGXMia5zhGr3/puzX9Kb1hWk1HMpaagayYYsuOHMcSoGkm/TO/xHcDQAT -FkLQBWENxMw3fCZ77CeiOj9HIZ+NCrvXPnqZsQq3xOJz8pJUH+O/c6xVXceXpatk -dZuwxHREv5nE+Z6Z8GTMLFNhHUCJ2OfyglPyJXQEo5aDGlf96RdB/qSX7hzCfPNi -h8zCxYn6AuwVOZqxh26SxXIRgA2kUS5UnqScLvIE8a5J1myS45J5DluV9wDGZHA6 -TAf/uQ== ------END CERTIFICATE----- diff --git a/server-key.pem b/server-key.pem deleted file mode 100644 index 75449e9146..0000000000 --- a/server-key.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJJwIBAAKCAgEAz5ww3zbFsi5t/PExS/0rv2zSiAdp5f3eR9IB8uXe3oE6TriZ -XIDqu3Hm2yd09EDp7/Rq7+2wt/ioYojutHNKzX8vIbN4WEv4h1dahaILhLlSoVkz -qEgPn5EJ7ZSHM97EJbjyzDnyGboQEc0uwW+We9Qurqf2lKh9QeagVON9lbRsWR+G -uoPh7YYreCT4T4r7qBDf95b1yAG79G5iutMpkKNkyDIKQL4nKWAq89MpC+VvBnIS -1iZ40XiyG8EAgyLhS3pstE9QmCt4jf5BE5iOohK0edZK9GM/BR+Suz1N6WA8L8s6 -44DjLmg8Q7ENtJMtr55YLIpqeh0LenhPA7OGnSOgslMoauXQAnU/LUP8mqx2bWDg -gzkNSanPf2w6VsSN79o5Bt2RTG0na0MFe1ibXUWsoui8UeITRvo6Srbx2MMy1rTo -NGKG+oO7ZfMghJuqkgy3lH0iiF/R/XeWo42TKqsbhOK3xIjFrUoC2OyghR0Ce6Xj -mVRPeyrJSyNayxmwpo1PhJRzQMvz7TtB9YaCltecZXPtwc7SSFW/lUNIdFE/E56O -xesJQZaDlSzZ/eYBfQIcnkuDnlNJ1IpWLtZXZz2Ir7gSH44FyD7e0Qeau4bkxekC -BLwDRubat2f5qKp8uE+6fk2FQkfa+cJw6rS4AM49CMXSoO/oWCxoQtXV/rUCAwEA -AQKCAgBff1cztfwmRBkv2rp0W9Z9zU7iZ5acxrQykTV4R6jG7OanQ164IEzZQEGS -2YmAuUNirylpiApb9bNu79Hti4MDF9fURru60edWSiZHhgyKVrzdnyyV4d/d5FeO -w12VzIipWXPF4Xmtv9qYdatmKzArcaSvjm9sXjyPzcg+hA5GiKG2NTxCG1wXaDoi -n87gt+uemXSMbRJNmA6fFw+vJGbgM2iudeYlddKJw8S+N4mQAU+axWpIu5WjtUMo -ZRUZPTgpAf1yy3ElLgncNvI0U154yuXdjWcZF3Xro2RZYI+wJCZLTc6ey7C0rhna -gVzujSzctZeVlW35Edcv1Jtz9maamPyxA9jVttLXdQkpw3yAP5lvYyacDND6s5Ob -QmJYs/ARhHDUCpWhh/PXUN1oIOqFxlJcJ9nD6HarDftuyZ9R6LkKlD9I8tIO3td+ -gvbKtdb+2BMf3559vGwHkl31bq9/Cb71n/3W+BlHkVoXfvvkMwh+2FX7weSw08i6 -84VQGwAQDi2xV1DK7leB9oHhLQh+bRw6+VeSpJe59twB/f+SBs6+WgoAnuE77JJ/ -xh/MWDCGQIzMpishmAiwK8Vt3HGXk2l3urRu2xsUHtA+xGqBaMsn76TZaCptxfhc -b8E6Dd+90bBIyLtleFjiHtrwMRgWNHcwa6xFONbmX0I4Lyv+WQKCAQEA+ckgD9TE -fnXgDvYUbpd987JTYePpYP+rE3EtnZLvwmG7GG+z57u3s/HJ2ltW/+tpMvNH5W5M -/i/L+1gRrs9R46kHfNxlWORyS5/U9LQRGru1nqyG2vCP1PRycizQ1rbx77GJraoG -Kgrg+cOshPOaISmtU2D0fpRunQEyE2E4guO3GUDkgxlc7WYVNMCE1O74aurPLVUI -6TSL3DNguaNzRwuhDBpTTS8+YDSP8JBTp4xIo/v8lnyxps1J2MF+OwKI+bTyc6AG -KRwqKwEjBZlPOam5WyhS5+gI4IGJ0OVr3ReKBy+mV0Y9U0Qtr4bM3l24EeoYJm9i -u725nyqjoB0pzwKCAQEA1MZzkX0LNOv9nPxbQKWRmtfHEztQDaVgLvM+yQTBFtpH -o41MaNGfaPLJo1I9xM1QGjIkSeQXhQuED0CLog9fc/QZ8Aj6r4ayq+smhGZBOcFs -StXpJUZSvJ0SYV5pW2S5ddj1zFoPxmGIcKNfNkYwRUv+F11uXkLl5OnSaCL7mTpS -Bsi9ebvVtkgAxRn+SBd8v6evQ0wKIUt6Wd9rbhD0SBnkzMZj1UEzWLKcm9OSOotM -CMYdGtBoaM2FEWzMLEK/AX7k/dMmSfn6y1L5kGfa6FLnxuSbP4gcPX6KGeAzOyFz -f4ByA+ZYGCaLvZygaP2BpPWJjmoujVTdgpYywU/kOwKCAQA0A4Swpz6vkr4wf1+r -09H/gB/K35nGgBbQup2kFWfl01IReZk95SIQz4WX+bM5IX1Lhctv5DMWLz/LvAAa -pcNIo/cSMhbMlu2R/9tphWGWboYYX5bJWpSo5Ko8SkbGn1vr7M1do20bLh7BA9UV -7mAnDbWxChx/i0spWIlj7bfF0cNb995dyldGfB5oWLqQWJ7ULMctimqLnUyOcwhu -P7IOlkZX7ddw2CimAewDTRqOZjDuLNfhQl//+vmosEon4ZRCY0KlfyopP1ssj1UK -om0T9drKN7Bj6DaJAZCANnXqmS895dc+jzrP7v+QcIKSv3NID7Ytsr4dmBliIo/m -FIyTAoIBADONrdy3Zmq5LybFQjrkouXYmESBn4De6/IUP8cm3r3xv1/SUYwoFi6w -fMiUmaDAekMOmRpKKKS0yJeFdQ+cF+2ZXFt7YXn78fiTri5wNCeRRPLQ539/jXZj -vFtZK7/YJNN38P5FaAMSCKP6AYy97t4rbnIs0hwGq3sOaEuRBlL2/X7lTntbVomm -4oDfLmw7PJ+XOnDc0KuGhSuXZYkylG2P0I8JFpFKDe3UwPtEhO6oZ5DMp6qiilmy -SiaRrNqghXUQUN1rNqC/79Mp1Iv3jeOyirP3GeKH0QfZTA6+8srF7R0cewOU4Q1g -8YTMmVU27x2P3HdkJMlboZwNVljeSZcCggEAe22CCvR/fQ0jlOeAoGUX+KO8AXSD -PQwVsB4IxtHBXgeIdrr0qNec6OIAJdmn1YnD0jA7dlQ2Kl08P7wR9/SJgCZZb8lJ -bW/4QziQIf8gDRJNDT/7YkQbSbdtdCsQN4DqVGHLoCgGwX/YuT4zzWywo5q95AM7 -Yyo7n6ehaQJOsG+2YoTswddyqhVN6r7pbJY3K1/fYLcIaHtDkbyX1QyMlnE1vIFg -MDYyLLbdQEORgTpEmA80HOjqujXlqwX3wQC9KJb16cBRfvz4kVh9smo8aFhVZj51 -wImXjw60CdkA6b3oyvmFhKETJnJl8AlzyfODbxQ59FC9j39TTSYhKwrnmA== ------END RSA PRIVATE KEY----- From c6d0d1b2028f2f4b6b23abda73e5b918a3bbe503 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 17 Jun 2023 16:52:22 +0200 Subject: [PATCH 008/114] Renamed Interface to Stream --- ...ttInterfacePlugin.java => MqttStreamPlugin.java} | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) rename plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/{MqttInterfacePlugin.java => MqttStreamPlugin.java} (96%) diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java similarity index 96% rename from plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java rename to plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index d2aa6714b3..c546b5c4bc 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttInterfacePlugin.java +++ b/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; -import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -47,14 +46,14 @@ import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; -public class MqttInterfacePlugin extends Plugin { +public class MqttStreamPlugin extends Plugin { /** * Constructor to be used by plugin manager for plugin instantiation. * Your plugins have to provide constructor with this exact signature to be successfully loaded by manager. */ - public MqttInterfacePlugin( PluginWrapper wrapper ) { + public MqttStreamPlugin( PluginWrapper wrapper ) { super( wrapper ); } @@ -65,19 +64,19 @@ public void start() { Map mqttSettings = new HashMap<>(); mqttSettings.put( "broker", "localhost" ); mqttSettings.put( "brokerPort", "1883" ); - QueryInterfaceManager.addInterfaceType( "mqtt", MqttInterfaceServer.class, mqttSettings ); + QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttSettings ); } @Override public void stop() { - QueryInterfaceManager.removeInterfaceType( MqttInterfaceServer.class ); + QueryInterfaceManager.removeInterfaceType( MqttStreamServer.class ); } @Slf4j @Extension - public static class MqttInterfaceServer extends QueryInterface { + public static class MqttStreamServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_NAME = "MQTT Interface"; @@ -103,7 +102,7 @@ public static class MqttInterfaceServer extends QueryInterface { private final MonitoringPage monitoringPage; - public MqttInterfaceServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { + public MqttStreamServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); //this.requestParser = new RequestParser( transactionManager, authenticator, "pa", "APP" ); this.uniqueName = uniqueName; From 5a2657236eed8ea9cf544901ce9f47e8299986b2 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 18 Jun 2023 10:15:58 +0200 Subject: [PATCH 009/114] Resolved issues coming from name changing of classes --- plugins/{mqtt-interface => mqtt-stream}/build.gradle | 0 plugins/{mqtt-interface => mqtt-stream}/gradle.properties | 6 +++--- .../main/java/org/polypheny/db/mqtt/MqttDocumentStore.java | 0 .../main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java | 5 ++++- settings.gradle | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) rename plugins/{mqtt-interface => mqtt-stream}/build.gradle (100%) rename plugins/{mqtt-interface => mqtt-stream}/gradle.properties (87%) rename plugins/{mqtt-interface => mqtt-stream}/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java (100%) rename plugins/{mqtt-interface => mqtt-stream}/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java (98%) diff --git a/plugins/mqtt-interface/build.gradle b/plugins/mqtt-stream/build.gradle similarity index 100% rename from plugins/mqtt-interface/build.gradle rename to plugins/mqtt-stream/build.gradle diff --git a/plugins/mqtt-interface/gradle.properties b/plugins/mqtt-stream/gradle.properties similarity index 87% rename from plugins/mqtt-interface/gradle.properties rename to plugins/mqtt-stream/gradle.properties index 6140d42ef2..90679de701 100644 --- a/plugins/mqtt-interface/gradle.properties +++ b/plugins/mqtt-stream/gradle.properties @@ -16,12 +16,12 @@ pluginVersion = 0.0.1 -pluginId = mqtt-interface -pluginClass = org.polypheny.db.mqtt.MqttInterfacePlugin +pluginId = mqtt-stream +pluginClass = org.polypheny.db.mqtt.MqttStreamPlugin pluginProvider = The Polypheny Project pluginDependencies = pluginUrlPath = -pluginCategories = interface +pluginCategories = other pluginPolyDependencies = pluginIsSystemComponent = false pluginIsUiVisible = true \ No newline at end of file diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java similarity index 100% rename from plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java rename to plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java diff --git a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java similarity index 98% rename from plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java rename to plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index c546b5c4bc..a4eb4f7d09 100644 --- a/plugins/mqtt-interface/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -37,10 +37,12 @@ import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.information.InformationAction; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; +import org.polypheny.db.information.InformationText; import org.polypheny.db.transaction.TransactionManager; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; @@ -315,11 +317,12 @@ public MonitoringPage() { InformationManager im = InformationManager.getInstance(); informationPage = new InformationPage(uniqueName, INTERFACE_NAME).fullWidth().setLabel("Interfaces"); - informationGroupTopics = new InformationGroup(informationPage, "Subscribed Topics"); + informationGroupTopics = new InformationGroup(informationPage, "Subscribed Topics").setOrder(1); im.addPage( informationPage ); im.addGroup( informationGroupTopics ); + // table to display topics topicsTable = new InformationTable( informationGroupTopics, List.of("Topics") diff --git a/settings.gradle b/settings.gradle index 3b3603f18a..5ed6815b90 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,7 +30,6 @@ include 'plugins:jdbc-adapter-framework' include 'plugins:avatica-interface' include 'plugins:rest-interface' include 'plugins:http-interface' -include 'plugins:mqtt-interface' // adapters plugins include 'plugins:hsqldb-adapter' @@ -49,6 +48,7 @@ include 'plugins:excel-adapter' // other plugins include 'plugins:explore-by-example' +include 'plugins:mqtt-stream' // disabled adapter plugins include 'plugins:pig-adapter' From 477ea760e61e7af910a319d433ea41334166aade Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 20 Jun 2023 17:05:33 +0200 Subject: [PATCH 010/114] changed the mqtt client library from eclipse paho to HiveMQ MQTT-Client --- plugins/mqtt-stream/build.gradle | 9 +- plugins/mqtt-stream/gradle.properties | 2 +- .../polypheny/db/mqtt/MqttDocumentStore.java | 4 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 261 +++++++----------- settings.gradle | 2 +- 5 files changed, 113 insertions(+), 165 deletions(-) diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 88b82db2ac..ca15092f25 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -6,8 +6,10 @@ dependencies { compileOnly project(":monitoring") // https://mvnrepository.com/artifact/org.eclipse.paho/org.eclipse.paho.client.mqttv3 - implementation group: 'org.eclipse.paho', name: 'org.eclipse.paho.client.mqttv3', version: eclipse_mqttv3_version + //implementation group: 'org.eclipse.paho', name: 'org.eclipse.paho.client.mqttv3', version: eclipse_mqttv3_version + // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client + implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: '1.3.1' // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") @@ -69,5 +71,8 @@ java { licensee { allow('Apache-2.0') - allow('MIT') + allowUrl('http://www.apache.org/license/LICENSE-2.0.txt') + allow('MIT-0') + + } diff --git a/plugins/mqtt-stream/gradle.properties b/plugins/mqtt-stream/gradle.properties index 90679de701..82bbbbe18d 100644 --- a/plugins/mqtt-stream/gradle.properties +++ b/plugins/mqtt-stream/gradle.properties @@ -21,7 +21,7 @@ pluginClass = org.polypheny.db.mqtt.MqttStreamPlugin pluginProvider = The Polypheny Project pluginDependencies = pluginUrlPath = -pluginCategories = other +pluginCategories = interface pluginPolyDependencies = pluginIsSystemComponent = false pluginIsUiVisible = true \ No newline at end of file diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java index 3d87c02c2e..d94ff9927e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java @@ -16,7 +16,7 @@ package org.polypheny.db.mqtt; -import org.eclipse.paho.client.mqttv3.MqttMessage; +//import org.eclipse.paho.client.mqttv3.MqttMessage; public class MqttDocumentStore { // should save the message in a document in the collection named as per the topic @@ -26,7 +26,7 @@ public void MqttDocumentStore() { } - public void saveMessage(String topic, MqttMessage message) { + public void saveMessage(String topic, String msg) { // TODO: implementation! // insert message in Document // and insert Document into Collection diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index a4eb4f7d09..0a7a9743dd 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -20,17 +20,14 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; + +import com.hivemq.client.mqtt.MqttClient; +import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import lombok.extern.slf4j.Slf4j; -import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; -import org.eclipse.paho.client.mqttv3.IMqttToken; -import org.eclipse.paho.client.mqttv3.IMqttActionListener; -import org.eclipse.paho.client.mqttv3.MqttCallback; -import org.eclipse.paho.client.mqttv3.MqttException; -import org.eclipse.paho.client.mqttv3.MqttMessage; -import org.eclipse.paho.client.mqttv3.MqttPersistenceException; import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; @@ -44,8 +41,8 @@ import org.polypheny.db.information.InformationTable; import org.polypheny.db.information.InformationText; import org.polypheny.db.transaction.TransactionManager; -import org.eclipse.paho.client.mqttv3.MqttAsyncClient; -import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; + +import static org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer.client; public class MqttStreamPlugin extends Plugin { @@ -63,10 +60,10 @@ public MqttStreamPlugin( PluginWrapper wrapper ) { @Override public void start() { // Add MQTT stream - Map mqttSettings = new HashMap<>(); - mqttSettings.put( "broker", "localhost" ); - mqttSettings.put( "brokerPort", "1883" ); - QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttSettings ); + Map mqttDefaultSettings = new HashMap<>(); + mqttDefaultSettings.put( "broker", "localhost" ); + mqttDefaultSettings.put( "brokerPort", "1883" ); + QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -83,7 +80,7 @@ public static class MqttStreamServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_NAME = "MQTT Interface"; @SuppressWarnings("WeakerAccess") - public static final String INTERFACE_DESCRIPTION = "Connection establishment to MQTT broker."; + public static final String INTERFACE_DESCRIPTION = "Connection establishment to a MQTT broker."; @SuppressWarnings("WeakerAccess") public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), @@ -95,11 +92,12 @@ public static class MqttStreamServer extends QueryInterface { private final String broker; - private final String brokerPort; + private final int brokerPort; - private ArrayList topics = new ArrayList(); + private List topics = new ArrayList(); - private MqttAsyncClient client; + public static Mqtt3AsyncClient client; + //private MqttAsyncClient client; private final MonitoringPage monitoringPage; @@ -111,75 +109,38 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au // Add information page monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); - brokerPort = settings.get( "brokerPort" ); - //subscribe(settings.get("topics")); + brokerPort = Integer.parseInt(settings.get( "brokerPort" )); } @Override public void run() { - String serverURI = String.format( "tcp://%s:%s", broker, brokerPort); - // creating fileStore to store all messages in this directory folder - //won't be needed later, when data is stored in data store - MqttDefaultFilePersistence fileStore = new MqttDefaultFilePersistence("C:\\Users\\Public\\UniProjekte\\BA_MQTT_Messages"); - try { - fileStore.open(uniqueName, serverURI); - } catch (MqttPersistenceException e) { - log.error( "There is a problem reading or writing persistence data." ); - } - try { - client = new MqttAsyncClient(serverURI, uniqueName, fileStore); - MqttCallback callback = new MqttCallback() { - @Override - public void connectionLost(Throwable cause) { - log.error( "Lost connection to the broker!" ); - //TODO: show this on UI! - } - - @Override - public void messageArrived(String topic, MqttMessage message) throws Exception { - log.info( "Message: {}", message.toString()); - MqttDocumentStore store = new MqttDocumentStore(); - store.saveMessage(topic, message); - //TODO: extract the important content of the message - // AND send it to StreamProcessor as PolyStream. - } - - @Override - public void deliveryComplete(IMqttDeliveryToken token) { - log.info( "Delivery of Message was successful!" ); - } - }; - - client.setCallback(callback); - IMqttActionListener connectionListener = new IMqttActionListener() { - @Override - public void onSuccess(IMqttToken asyncActionToken) { - //TODO: show on UI - log.info( "{} started and is listening to broker {}:{}", INTERFACE_NAME, broker, brokerPort ); - } - - @Override - public void onFailure(IMqttToken asyncActionToken, Throwable exception) { - //TODO: show on UI - log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); - } - }; - IMqttToken connToken = client.connect(null, connectionListener); - connToken.waitForCompletion(); - - - // Testing the connection: - //String str = "Hello, I am the Polypheny-Client!"; - //MqttMessage msg = new MqttMessage(str.getBytes()); - //IMqttToken pubToken= client.publish("testTopic", msg); - // Testing subscribtion: - //IMqttToken subToken= client.subscribe("testTopic", 1); + // commented code used for SSL connection + client = MqttClient.builder() + .useMqttVersion3() + .identifier( uniqueName ) + .serverHost( broker ) + .serverPort( brokerPort ) + //.useSslWithDefaultConfig() + .buildAsync(); + + client.connectWith() + //.simpleAuth() + //.username("my-user") + //.password("my-password".getBytes()) + //.applySimpleAuth() + .send() + .whenComplete((connAck, throwable) -> { + if (throwable != null) { + log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); + } else { + log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); + subscribeToAllTopics(Arrays.asList(settings.get( "topics" ).split( ", " ))); + } + } + ); - } catch (MqttException e) { - log.error( "An error occurred while communicating to the server."); - } } @Override @@ -190,106 +151,88 @@ public List getAvailableSettings() { @Override public void shutdown() { - IMqttActionListener shutDownListener = new IMqttActionListener() { - @Override - public void onSuccess(IMqttToken asyncActionToken) { - //TODO: show on UI - log.info( "{} stopped.", INTERFACE_NAME); - } - @Override - public void onFailure(IMqttToken asyncActionToken, Throwable exception) { - //TODO: show on UI - log.info( "{} could not disconnected from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); - } - }; - - try { - client.disconnect(null, shutDownListener); - } catch (MqttException e) { - log.error( "An error occurred while disconnecting from the broker {}:{}. Please try again.", broker, brokerPort ); - } - - monitoringPage.remove(); + client.disconnect().whenComplete((disconn, throwable)-> { + if(throwable != null) { + log.info( "{} could not disconnected from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + } else { + log.info( "{} stopped." , INTERFACE_NAME); + monitoringPage.remove(); + } + } + ); } + void subscribeToAllTopics(List newTopics) { + for ( String t : newTopics) { + subscribe( t ); + } + } - public void subscribe(String topic){ - //TODO: trigger from UI. - if (topics.isEmpty() || !topics.contains(topic)) { - IMqttActionListener subListener = new IMqttActionListener() { - @Override - public void onSuccess(IMqttToken asyncActionToken) { - topics.add(topic); - log.info( "Successfull subscription to {}.", topic ); - } + /** + * subcribes to one given topic and adds it to the List topics. + * @param topic the topic the client should subscribe to. + */ + public void subscribe(String topic) { + + if ( !topics.contains(topic) ) { + + client.subscribeWith() + .topicFilter(topic) + .callback(publish -> { + // TODO:Process the received message + log.info(String.format("Received Message from topic %s : %s.", topic, publish.getPayload())); + }) + .send() + .whenComplete((subAck, throwable) -> { + if (throwable != null) { + log.info("Subscription was not successfull. Please try again."); + } else { + topics.add(topic); + log.info("Successfull subscription to {}.", topic); + } + }).notifyAll(); /** notifys that subscription is finished. Now publish can be started.**/ - @Override - public void onFailure(IMqttToken asyncActionToken, Throwable exception) { - log.info( "Subscription was not successfull. Please try again." ); - } - }; - try { - IMqttToken subToken= client.subscribe(topic, 1, null, subListener); - subToken.waitForCompletion(); - } catch (MqttException e) { - log.error( "An error occurred while subscribing to {}. Please try again.", topic ); - } } + } public void unsubscribe(String topic) { - //TODO: trigger from UI. - if (topics.contains(topic)) { - IMqttActionListener unsubListener = new IMqttActionListener() { - @Override - public void onSuccess(IMqttToken asyncActionToken) { - topics.remove(topic); - log.info( "Successfull unsubscription from {}.", topic ); - } - @Override - public void onFailure(IMqttToken asyncActionToken, Throwable exception) { - log.info( "Unsubscription was not successfull. Please try again." ); - } - }; - try { - IMqttToken subToken= client.unsubscribe(topic, null, unsubListener); - } catch (MqttException e) { - log.error( "An error occurred while unsubscribing from {}. Please try again.", topic ); - } + if ( topics.contains(topic) ) { + client.unsubscribeWith().topicFilter(topic).send().whenComplete((unsub, throwable) -> { + if (throwable != null) { + log.error(String.format("Topic %s could not be unsubscribed.", topic)); + } else { + topics.remove(topic); + log.info( String.format("Unsubscribed from %s.", topic)); + } + } + ); } } - public void publish(String topic, String msg) { - //TODO: trigger from UI. - if (topics.contains(topic)) { - IMqttActionListener pubListener = new IMqttActionListener() { - @Override - public void onSuccess(IMqttToken asyncActionToken) { - log.info( "Message was published successfully."); + + @Override + protected void reloadSettings( List updatedSettings ) { + // updatedSettings only has the name of field that has been changed! + if (updatedSettings.contains("topics")) { + List newTopicsList = Arrays.asList(this.getCurrentSettings().get("topics").split( ", " )); + + for ( String newTopic : newTopicsList) { + if(!topics.contains(newTopic)) { + subscribe(newTopic); } + } - @Override - public void onFailure(IMqttToken asyncActionToken, Throwable exception) { - log.info( "Unsubscription was not successfull. Please try again." ); + for (String oldTopic : topics) { + if(!newTopicsList.contains(oldTopic)) { + unsubscribe(oldTopic); } - }; - MqttMessage message = new MqttMessage(msg.getBytes()); - try { - IMqttToken subToken= client.publish(topic, message, null, pubListener); - } catch (MqttException e) { - log.error( "An error occurred while unsubscribing from {}. Please try again.", topic ); } } - } - - - @Override - protected void reloadSettings( List updatedSettings ) { - // There is no modifiable setting for this query interface } diff --git a/settings.gradle b/settings.gradle index 5ed6815b90..93081340d2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,7 @@ include 'plugins:jdbc-adapter-framework' include 'plugins:avatica-interface' include 'plugins:rest-interface' include 'plugins:http-interface' +include 'plugins:mqtt-stream' // adapters plugins include 'plugins:hsqldb-adapter' @@ -48,7 +49,6 @@ include 'plugins:excel-adapter' // other plugins include 'plugins:explore-by-example' -include 'plugins:mqtt-stream' // disabled adapter plugins include 'plugins:pig-adapter' From 8721294b93ed055ec7efc0b964b29069b714381c Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 22 Jun 2023 10:53:24 +0200 Subject: [PATCH 011/114] added the mqtt client library version as variable also removed the eclipse library --- gradle.properties | 2 +- plugins/mqtt-stream/build.gradle | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/gradle.properties b/gradle.properties index 0131529122..5579dfc39e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -47,7 +47,6 @@ commons_text_version = 1.10.0 cottontaildb_version = 0.13.3 cottontaildb_driver_version = 0.13.0 cottontaildb_grpc_version = 1.36.0 -eclipse_mqttv3_version = 1.2.5 elasticsearch_rest_client_version = 6.2.4 elasticsearch_version = 6.2.4 embedded_monetdb_version = 2.39 @@ -70,6 +69,7 @@ h2_version = 1.4.197 hadoop_client_version = 2.7.5 hadoop_common_version = 2.7.5 hamcrest_core_version = 1.3 +hivemq_mqttclient_version = 1.3.1 hsqldb_version = 2.6.1 httpclient_version = 4.5.6 httpcore_version = 4.4.10 diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index ca15092f25..2286f228c0 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -5,11 +5,8 @@ dependencies { compileOnly project(":core") compileOnly project(":monitoring") - // https://mvnrepository.com/artifact/org.eclipse.paho/org.eclipse.paho.client.mqttv3 - //implementation group: 'org.eclipse.paho', name: 'org.eclipse.paho.client.mqttv3', version: eclipse_mqttv3_version - // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client - implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: '1.3.1' + implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: hivemq_mqttclient_version // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") From 2b888db0aa5a12088c4de23a65944c9c135eb81f Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 24 Jun 2023 15:59:02 +0200 Subject: [PATCH 012/114] implemented skeleton of StreamProcessing Method processMsg() and toString() implemented. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 50 ++++++++++++---- .../polypheny/db/mqtt/StreamProcessing.java | 60 +++++++++++++++++++ 2 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 0a7a9743dd..4a93838908 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -24,9 +24,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; +import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -88,8 +90,6 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingList( "topics", false, true, true, null ) ); - private final String uniqueName; - private final String broker; private final int brokerPort; @@ -105,7 +105,6 @@ public static class MqttStreamServer extends QueryInterface { public MqttStreamServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); //this.requestParser = new RequestParser( transactionManager, authenticator, "pa", "APP" ); - this.uniqueName = uniqueName; // Add information page monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); @@ -119,7 +118,7 @@ public void run() { // commented code used for SSL connection client = MqttClient.builder() .useMqttVersion3() - .identifier( uniqueName ) + .identifier( getUniqueName() ) .serverHost( broker ) .serverPort( brokerPort ) //.useSslWithDefaultConfig() @@ -154,9 +153,10 @@ public void shutdown() { client.disconnect().whenComplete((disconn, throwable)-> { if(throwable != null) { - log.info( "{} could not disconnected from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); } else { log.info( "{} stopped." , INTERFACE_NAME); + topics.clear(); monitoringPage.remove(); } } @@ -177,12 +177,12 @@ void subscribeToAllTopics(List newTopics) { public void subscribe(String topic) { if ( !topics.contains(topic) ) { - client.subscribeWith() .topicFilter(topic) - .callback(publish -> { + .callback(subMsg -> { // TODO:Process the received message - log.info(String.format("Received Message from topic %s : %s.", topic, publish.getPayload())); + log.info("Received Message from topic {} : {}.", subMsg.getTopic().toString(), subMsg.getPayload()); + processMsg(subMsg); }) .send() .whenComplete((subAck, throwable) -> { @@ -190,9 +190,10 @@ public void subscribe(String topic) { log.info("Subscription was not successfull. Please try again."); } else { topics.add(topic); - log.info("Successfull subscription to {}.", topic); + log.info("Successful subscription to topic {}.", topic); } - }).notifyAll(); /** notifys that subscription is finished. Now publish can be started.**/ + }); + //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). } @@ -213,28 +214,51 @@ public void unsubscribe(String topic) { } } - + @Override protected void reloadSettings( List updatedSettings ) { - // updatedSettings only has the name of field that has been changed! + // updatedSettings only has the name of fields that has been changed! if (updatedSettings.contains("topics")) { List newTopicsList = Arrays.asList(this.getCurrentSettings().get("topics").split( ", " )); for ( String newTopic : newTopicsList) { if(!topics.contains(newTopic)) { subscribe(newTopic); + log.info(String.format("subscribed to new topic: %s", newTopic)); } } for (String oldTopic : topics) { if(!newTopicsList.contains(oldTopic)) { unsubscribe(oldTopic); + log.info(String.format("unsubscribed form old topic: %s", oldTopic)); } } } } + void processMsg(Mqtt3Publish subMsg) { + String msg = StreamProcessing.processMsg(subMsg); + + + } + + /** + @Override + protected List applySettings( Map newSettings ) { + List updatedSettings = new ArrayList<>(); + for ( Map.Entry newSetting : newSettings.entrySet() ) { + if ( !Objects.equals( this.settings.get( newSetting.getKey() ), newSetting.getValue() ) ) { + this.settings.put( newSetting.getKey(), newSetting.getValue() ); + updatedSettings.add( newSetting.getKey() ); + } + } + + return updatedSettings; + } + **/ + @Override public void languageChange() { @@ -259,7 +283,7 @@ private class MonitoringPage { public MonitoringPage() { InformationManager im = InformationManager.getInstance(); - informationPage = new InformationPage(uniqueName, INTERFACE_NAME).fullWidth().setLabel("Interfaces"); + informationPage = new InformationPage(getUniqueName(), INTERFACE_NAME).fullWidth().setLabel("Interfaces"); informationGroupTopics = new InformationGroup(informationPage, "Subscribed Topics").setOrder(1); im.addPage( informationPage ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java new file mode 100644 index 0000000000..72ef114b3d --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import lombok.extern.slf4j.Slf4j; + +import java.io.UnsupportedEncodingException; +import java.nio.Buffer; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.NoSuchElementException; + +@Slf4j +public class StreamProcessing { + private Byte message; + // Message printed: Optional[java.nio.HeapByteBufferR[pos=0 lim=5 cap=5]]. + //new String( new byte[]{info}, "US-ASCII" ) + public static String processMsg(Mqtt3Publish subMsg) { + String msg = toString(subMsg); + String info = extractInfo(msg); + if (validateMsg(msg)) { + return msg; + } else { + log.error( "Message is not valid!" ); + return null; + } + } + + private static String toString(Mqtt3Publish subMsg) { + return new String(subMsg.getPayloadAsBytes(), Charset.defaultCharset()); + } + + private static boolean validateMsg(String msg) { + //TODO: Implement + return true; + } + + private static String extractInfo(String msg) { + //TODO: extract the needed Info only -> based on topic attribut on right side!!}; + return msg; + } + + +} From ec4b9b1089e819d9f9400ef9ee2482c322adfd46 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 25 Jun 2023 13:35:25 +0200 Subject: [PATCH 013/114] corrected minor things in processMsg method --- .../main/java/org/polypheny/db/mqtt/StreamProcessing.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java index 72ef114b3d..ca0e4fece6 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -28,14 +28,12 @@ @Slf4j public class StreamProcessing { - private Byte message; - // Message printed: Optional[java.nio.HeapByteBufferR[pos=0 lim=5 cap=5]]. - //new String( new byte[]{info}, "US-ASCII" ) + public static String processMsg(Mqtt3Publish subMsg) { String msg = toString(subMsg); String info = extractInfo(msg); - if (validateMsg(msg)) { - return msg; + if (validateMsg(info)) { + return info; } else { log.error( "Message is not valid!" ); return null; From e4953dca4d220d742035e873eea3185f0da21cb1 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 25 Jun 2023 17:14:01 +0200 Subject: [PATCH 014/114] implemented registerTopicFolder and saveMsgInFile methods saveMsgInFile method not done yet. Deleted unnecessary file which was created before. --- .../polypheny/db/mqtt/MqttDocumentStore.java | 39 ------ .../polypheny/db/mqtt/MqttStreamPlugin.java | 5 +- .../org/polypheny/db/mqtt/StreamCapture.java | 132 ++++++++++++++++++ 3 files changed, 135 insertions(+), 41 deletions(-) delete mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java deleted file mode 100644 index d94ff9927e..0000000000 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttDocumentStore.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.mqtt; - -//import org.eclipse.paho.client.mqttv3.MqttMessage; - -public class MqttDocumentStore { - // should save the message in a document in the collection named as per the topic - static String[] collections; - - public void MqttDocumentStore() { - - } - - public void saveMessage(String topic, String msg) { - // TODO: implementation! - // insert message in Document - // and insert Document into Collection - - } - - public void createCollection(String topic) { - //MqlCreateCollection collection = new MqlCreateCollection(null, topic, null); - } -} \ No newline at end of file diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 4a93838908..200f52fc5e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -181,7 +181,7 @@ public void subscribe(String topic) { .topicFilter(topic) .callback(subMsg -> { // TODO:Process the received message - log.info("Received Message from topic {} : {}.", subMsg.getTopic().toString(), subMsg.getPayload()); + log.info("Received message from topic {}.", subMsg.getTopic().toString()); processMsg(subMsg); }) .send() @@ -239,8 +239,9 @@ protected void reloadSettings( List updatedSettings ) { } void processMsg(Mqtt3Publish subMsg) { + //TODO: attention: return values, not correct, might need a change of type. String msg = StreamProcessing.processMsg(subMsg); - + StreamCapture.saveMsgInDocument(getUniqueName(), subMsg.getTopic().toString(), msg); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java new file mode 100644 index 0000000000..d960789a18 --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -0,0 +1,132 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import javafx.util.converter.LocalTimeStringConverter; +import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.transaction.TransactionManager; +import org.polypheny.db.util.PolyphenyHomeDirManager; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +@Slf4j +public class StreamCapture { + TransactionManager transactionManager; + PolyphenyHomeDirManager homeDirManager; + String namespace; + + StreamCapture (final TransactionManager transactionManager, String namespace) { + this.transactionManager = transactionManager; + this.namespace = namespace; + } + + + public static void saveMsgInDocument(String namespace, String topic, String message) { + String path = registerTopicFolder(topic); + saveMsgInFile(path, message); + createCollection(namespace, topic); + } + + + static void createCollection(String namespace, String topic) { + /** + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + AlgBuilder algBuilder = AlgBuilder.create( statement ); + JavaTypeFactory typeFactory = transaction.getTypeFactory(); + RexBuilder rexBuilder = new RexBuilder( typeFactory ); + + PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); + List names = List.of(namespace, ); + AlgOptTable collection = catalogReader.getCollection() + **/ + +/** + Catalog catalog = Catalog.getInstance(); + AdapterManager adapterManager = AdapterManager.getInstance(); + + long schemaId; + try { + schemaId = catalog.getSchema( Catalog.defaultDatabaseId, ((MqlQueryParameters) parameters).getDatabase() ).id; + } catch ( UnknownSchemaException e ) { + throw new RuntimeException( "The used document database (Polypheny Schema) is not available." ); + } + + Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; + + try { + List dataStores = stores + .stream() + .map( store -> (DataStore) adapterManager.getAdapter( store ) ) + .collect( Collectors.toList() ); + DdlManager.getInstance().createCollection( + schemaId, + topic, + true, + dataStores.size() == 0 ? null : dataStores, + placementType, + statement ); + } catch ( EntityAlreadyExistsException e ) { + throw new RuntimeException( "The generation of the collection was not possible, due to: " + e.getMessage() ); + } + **/ + + } + + private static String registerTopicFolder(String topic) { + PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); + + String path = File.separator + "mqttStreamPlugin" + File.separator + topic.replace("/", File.separator);; + + File file = null; + if ( !homeDirManager.checkIfExists(path) ) { + file = homeDirManager.registerNewFolder(path); + log.info( "New Directory created!" ); + } else { + //TODO: rmv log + log.info("Directory already exists"); + } + + return path; + + } + + static boolean saveMsgInFile (String path, String msg) { + PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); + String filePath = path + File.separator + System.currentTimeMillis() + ".txt"; + File msgFile = homeDirManager.registerNewFile(filePath); + if ( msgFile.canWrite() ) { + //msgFile. + } else { + log.error( "Cannot write write in " ); + } + return true; + } + + + private void getTransaction() { + /** + try { + return transactionManager.startTransaction( userId, databaseId, false, "Stream Processing", Transaction.MultimediaFlavor.FILE ); + } catch (UnknownUserException | UnknownDatabaseException | UnknownSchemaException e ) { + throw new RuntimeException( "Error while starting transaction", e ); + } + **/ + } +} From 9684d692bdab6df52ca6b38581535ec4aec77f89 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 27 Jun 2023 13:03:19 +0200 Subject: [PATCH 015/114] implemented createCollection method works, but aspect with namespace comparison is still missing --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 198 +++++++++--------- .../org/polypheny/db/mqtt/StreamCapture.java | 106 +++++----- 2 files changed, 149 insertions(+), 155 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 200f52fc5e..0659d8fb84 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -18,17 +18,14 @@ import com.google.common.collect.ImmutableList; - +import com.hivemq.client.mqtt.MqttClient; +import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; +import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Consumer; - -import com.hivemq.client.mqtt.MqttClient; -import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; -import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -36,16 +33,12 @@ import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; -import org.polypheny.db.information.InformationAction; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; -import org.polypheny.db.information.InformationText; import org.polypheny.db.transaction.TransactionManager; -import static org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer.client; - public class MqttStreamPlugin extends Plugin { @@ -87,7 +80,8 @@ public static class MqttStreamServer extends QueryInterface { public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), - new QueryInterfaceSettingList( "topics", false, true, true, null ) + new QueryInterfaceSettingString( "namespace", false, true, false, null ), + new QueryInterfaceSettingString( "topics", false, true, true, null ) ); private final String broker; @@ -97,7 +91,8 @@ public static class MqttStreamServer extends QueryInterface { private List topics = new ArrayList(); public static Mqtt3AsyncClient client; - //private MqttAsyncClient client; + + public final String namespace; private final MonitoringPage monitoringPage; @@ -108,7 +103,11 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au // Add information page monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); - brokerPort = Integer.parseInt(settings.get( "brokerPort" )); + brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); + namespace = settings.get( "namespace" ); + //TODO: rmv + log.info( "MqttStreamPlugin constructor was called!" ); + } @@ -117,31 +116,32 @@ public void run() { // commented code used for SSL connection client = MqttClient.builder() - .useMqttVersion3() - .identifier( getUniqueName() ) - .serverHost( broker ) - .serverPort( brokerPort ) - //.useSslWithDefaultConfig() - .buildAsync(); + .useMqttVersion3() + .identifier( getUniqueName() ) + .serverHost( broker ) + .serverPort( brokerPort ) + //.useSslWithDefaultConfig() + .buildAsync(); client.connectWith() - //.simpleAuth() - //.username("my-user") - //.password("my-password".getBytes()) - //.applySimpleAuth() - .send() - .whenComplete((connAck, throwable) -> { - if (throwable != null) { - log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); - } else { - log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - subscribeToAllTopics(Arrays.asList(settings.get( "topics" ).split( ", " ))); - } - } - ); + //.simpleAuth() + //.username("my-user") + //.password("my-password".getBytes()) + //.applySimpleAuth() + .send() + .whenComplete( ( connAck, throwable ) -> { + if ( throwable != null ) { + log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); + } else { + log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); + subscribeToAllTopics( Arrays.asList( settings.get( "topics" ).trim().split( "," )) ); + } + } + ); } + @Override public List getAvailableSettings() { return AVAILABLE_SETTINGS; @@ -151,12 +151,12 @@ public List getAvailableSettings() { @Override public void shutdown() { - client.disconnect().whenComplete((disconn, throwable)-> { - if(throwable != null) { + client.disconnect().whenComplete( ( disconn, throwable ) -> { + if ( throwable != null ) { log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); } else { - log.info( "{} stopped." , INTERFACE_NAME); - topics.clear(); + log.info( "{} stopped.", INTERFACE_NAME ); + //topics.clear(); monitoringPage.remove(); } } @@ -164,50 +164,54 @@ public void shutdown() { } - void subscribeToAllTopics(List newTopics) { - for ( String t : newTopics) { + + void subscribeToAllTopics( List newTopics ) { + for ( String t : newTopics ) { subscribe( t ); } } + /** - * subcribes to one given topic and adds it to the List topics. + * subscribes to one given topic and adds it to the List topics. + * * @param topic the topic the client should subscribe to. */ - public void subscribe(String topic) { + public void subscribe( String topic ) { - if ( !topics.contains(topic) ) { + if ( !topics.contains( topic ) ) { client.subscribeWith() - .topicFilter(topic) - .callback(subMsg -> { - // TODO:Process the received message - log.info("Received message from topic {}.", subMsg.getTopic().toString()); - processMsg(subMsg); - }) - .send() - .whenComplete((subAck, throwable) -> { - if (throwable != null) { - log.info("Subscription was not successfull. Please try again."); - } else { - topics.add(topic); - log.info("Successful subscription to topic {}.", topic); - } - }); + .topicFilter( topic ) + .callback( subMsg -> { + // TODO:Process the received message + log.info( "Received message from topic {}.", subMsg.getTopic().toString() ); + processMsg( subMsg ); + } ) + .send() + .whenComplete( ( subAck, throwable ) -> { + if ( throwable != null ) { + log.info( "Subscription was not successfull. Please try again." ); + } else { + topics.add( topic ); + log.info( "Successful subscription to topic {}.", topic ); + } + } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). } } - public void unsubscribe(String topic) { - if ( topics.contains(topic) ) { - client.unsubscribeWith().topicFilter(topic).send().whenComplete((unsub, throwable) -> { - if (throwable != null) { - log.error(String.format("Topic %s could not be unsubscribed.", topic)); + public void unsubscribe( String topic ) { + + if ( topics.contains( topic ) ) { + client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { + if ( throwable != null ) { + log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { - topics.remove(topic); - log.info( String.format("Unsubscribed from %s.", topic)); + topics.remove( topic ); + log.info( String.format( "Unsubscribed from %s.", topic ) ); } } ); @@ -218,46 +222,46 @@ public void unsubscribe(String topic) { @Override protected void reloadSettings( List updatedSettings ) { // updatedSettings only has the name of fields that has been changed! - if (updatedSettings.contains("topics")) { - List newTopicsList = Arrays.asList(this.getCurrentSettings().get("topics").split( ", " )); + if ( updatedSettings.contains( "topics" ) ) { + List newTopicsList = Arrays.asList( this.getCurrentSettings().get( "topics" ).trim().split( "," ) ); - for ( String newTopic : newTopicsList) { - if(!topics.contains(newTopic)) { - subscribe(newTopic); - log.info(String.format("subscribed to new topic: %s", newTopic)); + for ( String newTopic : newTopicsList ) { + if ( !topics.contains( newTopic ) ) { + subscribe( newTopic ); } } - for (String oldTopic : topics) { - if(!newTopicsList.contains(oldTopic)) { - unsubscribe(oldTopic); - log.info(String.format("unsubscribed form old topic: %s", oldTopic)); + for ( String oldTopic : topics ) { + if ( !newTopicsList.contains( oldTopic ) ) { + unsubscribe( oldTopic ); } } } } - void processMsg(Mqtt3Publish subMsg) { + + void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. - String msg = StreamProcessing.processMsg(subMsg); - StreamCapture.saveMsgInDocument(getUniqueName(), subMsg.getTopic().toString(), msg); + String msg = StreamProcessing.processMsg( subMsg ); + StreamCapture streamCapture = new StreamCapture( this.transactionManager, this.namespace ); + streamCapture.saveMsgInDocument( subMsg.getTopic().toString(), msg ); } - /** - @Override - protected List applySettings( Map newSettings ) { - List updatedSettings = new ArrayList<>(); - for ( Map.Entry newSetting : newSettings.entrySet() ) { - if ( !Objects.equals( this.settings.get( newSetting.getKey() ), newSetting.getValue() ) ) { - this.settings.put( newSetting.getKey(), newSetting.getValue() ); - updatedSettings.add( newSetting.getKey() ); - } - } - return updatedSettings; - } + /** + * @Override protected List applySettings( Map newSettings ) { + * List updatedSettings = new ArrayList<>(); + * for ( Map.Entry newSetting : newSettings.entrySet() ) { + * if ( !Objects.equals( this.settings.get( newSetting.getKey() ), newSetting.getValue() ) ) { + * this.settings.put( newSetting.getKey(), newSetting.getValue() ); + * updatedSettings.add( newSetting.getKey() ); + * } + * } + * + * return updatedSettings; + * } **/ @@ -281,11 +285,12 @@ private class MonitoringPage { private InformationTable topicsTable; + public MonitoringPage() { InformationManager im = InformationManager.getInstance(); - informationPage = new InformationPage(getUniqueName(), INTERFACE_NAME).fullWidth().setLabel("Interfaces"); - informationGroupTopics = new InformationGroup(informationPage, "Subscribed Topics").setOrder(1); + informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).fullWidth().setLabel( "Interfaces" ); + informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 1 ); im.addPage( informationPage ); im.addGroup( informationGroupTopics ); @@ -293,7 +298,7 @@ public MonitoringPage() { // table to display topics topicsTable = new InformationTable( informationGroupTopics, - List.of("Topics") + List.of( "Topics" ) ); im.registerInformation( topicsTable ); @@ -303,17 +308,18 @@ public MonitoringPage() { public void update() { - if( topics.isEmpty() ) { - topicsTable.addRow("No topic subscriptions"); + if ( topics.isEmpty() ) { + topicsTable.addRow( "No topic subscriptions" ); } else { topicsTable.reset(); - for( String topic: topics) { - topicsTable.addRow(topic); + for ( String topic : topics ) { + topicsTable.addRow( topic ); } } } + public void remove() { InformationManager im = InformationManager.getInstance(); im.removeInformation( topicsTable ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index d960789a18..87b1b404cb 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -16,117 +16,105 @@ package org.polypheny.db.mqtt; -import javafx.util.converter.LocalTimeStringConverter; +import java.io.File; +import java.util.ArrayList; +import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.adapter.AdapterManager; +import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; +import org.polypheny.db.ddl.DdlManager; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; import org.polypheny.db.util.PolyphenyHomeDirManager; -import java.io.File; -import java.io.IOException; -import java.util.List; - @Slf4j public class StreamCapture { + TransactionManager transactionManager; PolyphenyHomeDirManager homeDirManager; String namespace; - StreamCapture (final TransactionManager transactionManager, String namespace) { + + StreamCapture( final TransactionManager transactionManager, String namespace ) { this.transactionManager = transactionManager; this.namespace = namespace; } - public static void saveMsgInDocument(String namespace, String topic, String message) { - String path = registerTopicFolder(topic); - saveMsgInFile(path, message); - createCollection(namespace, topic); + public void saveMsgInDocument( String topic, String message ) { + //String path = registerTopicFolder(topic); + this.createCollection( topic ); + //TODO: create Document in collection with message } - static void createCollection(String namespace, String topic) { - /** - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - AlgBuilder algBuilder = AlgBuilder.create( statement ); - JavaTypeFactory typeFactory = transaction.getTypeFactory(); - RexBuilder rexBuilder = new RexBuilder( typeFactory ); - - PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); - List names = List.of(namespace, ); - AlgOptTable collection = catalogReader.getCollection() - **/ - -/** + void createCollection( String topic ) { Catalog catalog = Catalog.getInstance(); - AdapterManager adapterManager = AdapterManager.getInstance(); long schemaId; - try { - schemaId = catalog.getSchema( Catalog.defaultDatabaseId, ((MqlQueryParameters) parameters).getDatabase() ).id; - } catch ( UnknownSchemaException e ) { - throw new RuntimeException( "The used document database (Polypheny Schema) is not available." ); - } + //TODO: evtl. defaultDatabaseID anpassen + getSchema methodenaufruf + schemaId = catalog.getSchema( Catalog.defaultDatabaseId ).id; Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; + Transaction transaction = null; + Statement statement = null; try { - List dataStores = stores - .stream() - .map( store -> (DataStore) adapterManager.getAdapter( store ) ) - .collect( Collectors.toList() ); + transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, Catalog.defaultDatabaseId, false, "MQTT Stream" ); + statement = transaction.createStatement(); + } catch ( Exception e ) { + log.error( "An error occurred: {}", e.getMessage() ); + } + + try { + List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( schemaId, - topic, - true, + this.namespace, + true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, placementType, statement ); + log.info( "Created Collection with name: {}", topic ); } catch ( EntityAlreadyExistsException e ) { throw new RuntimeException( "The generation of the collection was not possible, due to: " + e.getMessage() ); } - **/ + } - private static String registerTopicFolder(String topic) { + + private static String registerTopicFolder( String topic ) { PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); - String path = File.separator + "mqttStreamPlugin" + File.separator + topic.replace("/", File.separator);; + String path = File.separator + "mqttStreamPlugin" + File.separator + topic.replace( "/", File.separator ); + ; File file = null; - if ( !homeDirManager.checkIfExists(path) ) { - file = homeDirManager.registerNewFolder(path); + if ( !homeDirManager.checkIfExists( path ) ) { + file = homeDirManager.registerNewFolder( path ); log.info( "New Directory created!" ); } else { //TODO: rmv log - log.info("Directory already exists"); + log.info( "Directory already exists" ); } return path; } - static boolean saveMsgInFile (String path, String msg) { - PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); - String filePath = path + File.separator + System.currentTimeMillis() + ".txt"; - File msgFile = homeDirManager.registerNewFile(filePath); - if ( msgFile.canWrite() ) { - //msgFile. - } else { - log.error( "Cannot write write in " ); - } - return true; - } - private void getTransaction() { /** - try { - return transactionManager.startTransaction( userId, databaseId, false, "Stream Processing", Transaction.MultimediaFlavor.FILE ); - } catch (UnknownUserException | UnknownDatabaseException | UnknownSchemaException e ) { - throw new RuntimeException( "Error while starting transaction", e ); - } + try { + return transactionManager.startTransaction( userId, databaseId, false, "Stream Processing", Transaction.MultimediaFlavor.FILE ); + } catch (UnknownUserException | UnknownDatabaseException | UnknownSchemaException e ) { + throw new RuntimeException( "Error while starting transaction", e ); + } **/ } + } From f8b55594cc8076b88263001f24cd42f8dbfe26ac Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 27 Jun 2023 14:49:44 +0200 Subject: [PATCH 016/114] resolved problem with inserting changed settings to map settings --- .../java/org/polypheny/db/iface/QueryInterface.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java index afa8a50e5c..0f56938101 100644 --- a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java +++ b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java @@ -19,11 +19,9 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; + import lombok.AllArgsConstructor; import lombok.Getter; import org.pf4j.ExtensionPoint; @@ -61,10 +59,15 @@ public QueryInterface( this.authenticator = authenticator; this.queryInterfaceId = queryInterfaceId; this.uniqueName = uniqueName; - this.settings = settings; + //this.settings = settings; this.supportsDml = supportsDml; this.supportsDdl = supportsDdl; + this.settings = new HashMap<>(settings.size()); + for ( Map.Entry entry : settings.entrySet()) { + this.settings.put(entry.getKey(), entry.getValue()); + } + LanguageManager.getINSTANCE().addObserver( this ); } From 27b96bae7c68b177e19d9cea43ae7a78a73c865d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 27 Jun 2023 14:50:18 +0200 Subject: [PATCH 017/114] Reformatted code --- .../polypheny/db/mqtt/StreamProcessing.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java index ca0e4fece6..2d61e3effe 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -17,22 +17,18 @@ package org.polypheny.db.mqtt; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; -import lombok.extern.slf4j.Slf4j; - -import java.io.UnsupportedEncodingException; -import java.nio.Buffer; -import java.nio.BufferUnderflowException; -import java.nio.ByteBuffer; import java.nio.charset.Charset; -import java.util.NoSuchElementException; +import lombok.extern.slf4j.Slf4j; @Slf4j public class StreamProcessing { - public static String processMsg(Mqtt3Publish subMsg) { - String msg = toString(subMsg); - String info = extractInfo(msg); - if (validateMsg(info)) { + public static String processMsg( Mqtt3Publish subMsg ) { + String msg = toString( subMsg ); + + String info = extractInfo( msg ); + if ( validateMsg( info ) ) { + log.info( "Extracted and validated message: {}", msg); return info; } else { log.error( "Message is not valid!" ); @@ -40,16 +36,19 @@ public static String processMsg(Mqtt3Publish subMsg) { } } - private static String toString(Mqtt3Publish subMsg) { - return new String(subMsg.getPayloadAsBytes(), Charset.defaultCharset()); + + private static String toString( Mqtt3Publish subMsg ) { + return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); } - private static boolean validateMsg(String msg) { + + private static boolean validateMsg( String msg ) { //TODO: Implement return true; } - private static String extractInfo(String msg) { + + private static String extractInfo( String msg ) { //TODO: extract the needed Info only -> based on topic attribut on right side!!}; return msg; } From d5c93e4d1b406f7f615113ca8ae2729cba90770f Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 29 Jun 2023 14:45:23 +0200 Subject: [PATCH 018/114] optimized and debugged reloadSettings method sub and unsub of topics after starting plugin works. "permanent" topics can't be unsubscribed permanently yet. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 144 +++++++++--------- 1 file changed, 69 insertions(+), 75 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 0659d8fb84..234fa9df69 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -23,9 +23,14 @@ import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -88,7 +93,7 @@ public static class MqttStreamServer extends QueryInterface { private final int brokerPort; - private List topics = new ArrayList(); + private ArrayList topics = new ArrayList(); public static Mqtt3AsyncClient client; @@ -99,15 +104,11 @@ public static class MqttStreamServer extends QueryInterface { public MqttStreamServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); - //this.requestParser = new RequestParser( transactionManager, authenticator, "pa", "APP" ); // Add information page monitoringPage = new MonitoringPage(); broker = settings.get( "broker" ); brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); namespace = settings.get( "namespace" ); - //TODO: rmv - log.info( "MqttStreamPlugin constructor was called!" ); - } @@ -134,7 +135,8 @@ public void run() { log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - subscribeToAllTopics( Arrays.asList( settings.get( "topics" ).trim().split( "," )) ); + + subscribe( Arrays.asList( settings.get( "topics" ).trim().split( "," )) ); } } ); @@ -165,7 +167,41 @@ public void shutdown() { } - void subscribeToAllTopics( List newTopics ) { + + @Override + protected void reloadSettings( List updatedSettings ) { + + if ( updatedSettings.contains( "topics" ) ) { + List newTopicsList = Stream.of(this.getCurrentSettings().get( "topics" ).split( "," )).map(String::trim).collect( Collectors.toList()); + + List topicsToSub = new ArrayList<>(); + for ( String newTopic : newTopicsList ) { + if ( !topics.contains( newTopic ) ) { + topicsToSub.add( newTopic ); + } + } + + if ( !topicsToSub.isEmpty()) { + subscribe( topicsToSub ); + } + + + List topicsToUnsub = new ArrayList<>(); + for ( String oldTopic : topics ) { + if ( !newTopicsList.contains( oldTopic ) ) { + topicsToUnsub.add( oldTopic ); + } + } + + if ( !topicsToUnsub.isEmpty() ) { + unsubscribe( topicsToUnsub ); + } + } + + } + + + void subscribe( List newTopics ) { for ( String t : newTopics ) { subscribe( t ); } @@ -178,69 +214,42 @@ void subscribeToAllTopics( List newTopics ) { * @param topic the topic the client should subscribe to. */ public void subscribe( String topic ) { - - if ( !topics.contains( topic ) ) { - client.subscribeWith() - .topicFilter( topic ) - .callback( subMsg -> { - // TODO:Process the received message - log.info( "Received message from topic {}.", subMsg.getTopic().toString() ); - processMsg( subMsg ); - } ) - .send() - .whenComplete( ( subAck, throwable ) -> { - if ( throwable != null ) { - log.info( "Subscription was not successfull. Please try again." ); - } else { - topics.add( topic ); - log.info( "Successful subscription to topic {}.", topic ); - } - } ); - //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). - - } + client.subscribeWith().topicFilter( topic ).callback( subMsg -> { + log.info( "Received message from topic {}.", subMsg.getTopic() ); + processMsg( subMsg ); + } ).send().whenComplete( ( subAck, throwable ) -> { + if ( throwable != null ) { + log.info( "Subscription was not successfull. Please try again." ); + } else { + this.topics.add( topic ); + log.info( "Successful subscription to topic {}.", topic ); + } + } ); + //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). } - - public void unsubscribe( String topic ) { - - if ( topics.contains( topic ) ) { - client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { - if ( throwable != null ) { - log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); - } else { - topics.remove( topic ); - log.info( String.format( "Unsubscribed from %s.", topic ) ); - } - } - ); + public void unsubscribe( List topics ) { + for ( String t : topics ) { + unsubscribe( t ); } } - @Override - protected void reloadSettings( List updatedSettings ) { - // updatedSettings only has the name of fields that has been changed! - if ( updatedSettings.contains( "topics" ) ) { - List newTopicsList = Arrays.asList( this.getCurrentSettings().get( "topics" ).trim().split( "," ) ); - - for ( String newTopic : newTopicsList ) { - if ( !topics.contains( newTopic ) ) { - subscribe( newTopic ); - } - } + public void unsubscribe( String topic ) { + client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { + if ( throwable != null ) { - for ( String oldTopic : topics ) { - if ( !newTopicsList.contains( oldTopic ) ) { - unsubscribe( oldTopic ); - } + log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); + } else { + this.topics.remove( topic ); + log.info( String.format( "Unsubscribed from %s.", topic ) ); } - } - + } ); } + void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. String msg = StreamProcessing.processMsg( subMsg ); @@ -250,21 +259,6 @@ void processMsg( Mqtt3Publish subMsg ) { } - /** - * @Override protected List applySettings( Map newSettings ) { - * List updatedSettings = new ArrayList<>(); - * for ( Map.Entry newSetting : newSettings.entrySet() ) { - * if ( !Objects.equals( this.settings.get( newSetting.getKey() ), newSetting.getValue() ) ) { - * this.settings.put( newSetting.getKey(), newSetting.getValue() ); - * updatedSettings.add( newSetting.getKey() ); - * } - * } - * - * return updatedSettings; - * } - **/ - - @Override public void languageChange() { @@ -283,7 +277,7 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; - private InformationTable topicsTable; + private final InformationTable topicsTable; public MonitoringPage() { @@ -308,10 +302,10 @@ public MonitoringPage() { public void update() { + topicsTable.reset(); if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - topicsTable.reset(); for ( String topic : topics ) { topicsTable.addRow( topic ); } From 38035fff54867753c76a66407c3efeed4796f968 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 29 Jun 2023 18:23:52 +0200 Subject: [PATCH 019/114] created skeleton of class PolyStream also reformatted code --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 18 ++++++------- .../org/polypheny/db/mqtt/PolyStream.java | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 234fa9df69..00daa78931 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -23,14 +23,12 @@ import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -97,6 +95,7 @@ public static class MqttStreamServer extends QueryInterface { public static Mqtt3AsyncClient client; + @Getter public final String namespace; private final MonitoringPage monitoringPage; @@ -116,7 +115,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au public void run() { // commented code used for SSL connection - client = MqttClient.builder() + this.client = MqttClient.builder() .useMqttVersion3() .identifier( getUniqueName() ) .serverHost( broker ) @@ -136,7 +135,7 @@ public void run() { } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - subscribe( Arrays.asList( settings.get( "topics" ).trim().split( "," )) ); + subscribe( Arrays.asList( settings.get( "topics" ).trim().split( "," ) ) ); } } ); @@ -167,12 +166,11 @@ public void shutdown() { } - @Override protected void reloadSettings( List updatedSettings ) { if ( updatedSettings.contains( "topics" ) ) { - List newTopicsList = Stream.of(this.getCurrentSettings().get( "topics" ).split( "," )).map(String::trim).collect( Collectors.toList()); + List newTopicsList = Stream.of( this.getCurrentSettings().get( "topics" ).split( "," ) ).map( String::trim ).collect( Collectors.toList() ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { @@ -181,11 +179,10 @@ protected void reloadSettings( List updatedSettings ) { } } - if ( !topicsToSub.isEmpty()) { + if ( !topicsToSub.isEmpty() ) { subscribe( topicsToSub ); } - List topicsToUnsub = new ArrayList<>(); for ( String oldTopic : topics ) { if ( !newTopicsList.contains( oldTopic ) ) { @@ -229,6 +226,7 @@ public void subscribe( String topic ) { } + public void unsubscribe( List topics ) { for ( String t : topics ) { unsubscribe( t ); @@ -249,8 +247,8 @@ public void unsubscribe( String topic ) { } - void processMsg( Mqtt3Publish subMsg ) { + // create Instance of PolyStream for the message with namespace //TODO: attention: return values, not correct, might need a change of type. String msg = StreamProcessing.processMsg( subMsg ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, this.namespace ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java new file mode 100644 index 0000000000..1cead079e9 --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +public class PolyStream { + + // Representation of 1 message in a namespace + String topic; + String message; + String namespace; + +} From 45fe6ede1d95b38bba19b630aa19973983367c58 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 29 Jun 2023 18:37:34 +0200 Subject: [PATCH 020/114] added namespaces to createCollection namespaces are now queried and created if a namespace of type Document is not existing. --- .../org/polypheny/db/mqtt/StreamCapture.java | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 87b1b404cb..ac808cdb7f 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -19,11 +19,18 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.AdapterManager; import org.polypheny.db.adapter.DataStore; import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.PlacementType; +import org.polypheny.db.catalog.entity.CatalogCollection; +import org.polypheny.db.catalog.entity.CatalogSchema; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; +import org.polypheny.db.catalog.exceptions.NamespaceAlreadyExistsException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; @@ -46,24 +53,53 @@ public class StreamCapture { public void saveMsgInDocument( String topic, String message ) { //String path = registerTopicFolder(topic); - this.createCollection( topic ); + long schemaId = createCollection( topic ); + log.info( "schemaId = {}", schemaId ); + log.info( "testing method createCollection" ); + Catalog catalog = Catalog.getInstance(); + CatalogSchema schema = null; + schema = catalog.getSchema( schemaId ); + + log.info( "Namespace name: {}, Namespace Type: {}, Database name: {}", schema.getName(), schema.getNamespaceType(), schema.getDatabaseName() ); //TODO: create Document in collection with message } + long createCollection( String topic ) { + return createCollection( topic, Catalog.defaultDatabaseId, Catalog.defaultUserId ); + } + long createCollection( String topic, long databaseId ) { + return createCollection( topic, databaseId, Catalog.defaultUserId ); + } + long createCollection( String topic, int userId ) { + return createCollection( topic, Catalog.defaultDatabaseId, userId ); + } + long createCollection( String topic, long databaseId, int userId ) { - void createCollection( String topic ) { Catalog catalog = Catalog.getInstance(); - long schemaId; - //TODO: evtl. defaultDatabaseID anpassen + getSchema methodenaufruf - schemaId = catalog.getSchema( Catalog.defaultDatabaseId ).id; + // In case there is a namespace with the same name existing and is of Type Document, then that id of the existing schema will be used to create a collection. + long schemaId = schemaId = catalog.addNamespace( this.namespace, databaseId, userId, NamespaceType.DOCUMENT ); + + // check for namespace and NamespaceType: + try { + CatalogSchema schema = catalog.getSchema( databaseId, this.namespace ); + + if ( catalog.checkIfExistsSchema( databaseId, this.namespace ) && schema.namespaceType == NamespaceType.DOCUMENT ) { + schemaId = schema.id; + } + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + } + + //TODO: abfragen ob es collection schon gibt + //TODO: Namespace abfragen/erstellen und collection abfragen/erstellen in versch methoden machen Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; Transaction transaction = null; Statement statement = null; try { - transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, Catalog.defaultDatabaseId, false, "MQTT Stream" ); + transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, databaseId, false, "MQTT Stream" ); statement = transaction.createStatement(); } catch ( Exception e ) { log.error( "An error occurred: {}", e.getMessage() ); @@ -73,16 +109,16 @@ void createCollection( String topic ) { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( schemaId, - this.namespace, + topic, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, placementType, statement ); log.info( "Created Collection with name: {}", topic ); } catch ( EntityAlreadyExistsException e ) { - throw new RuntimeException( "The generation of the collection was not possible, due to: " + e.getMessage() ); + log.error( "The generation of the collection was not possible due to: " + e.getMessage() ); } - + return schemaId; } @@ -107,14 +143,4 @@ private static String registerTopicFolder( String topic ) { } - private void getTransaction() { - /** - try { - return transactionManager.startTransaction( userId, databaseId, false, "Stream Processing", Transaction.MultimediaFlavor.FILE ); - } catch (UnknownUserException | UnknownDatabaseException | UnknownSchemaException e ) { - throw new RuntimeException( "Error while starting transaction", e ); - } - **/ - } - } From 097d3a04bccf62e53978a2c00ae44b09cae11a98 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 2 Jul 2023 17:40:11 +0200 Subject: [PATCH 021/114] implemented the process to get a collection All aspects (namespaces, collections, collection type, placement) are now considered. Nothing tested yet... (therefore created main method). Also separated creating namespace and collection. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 6 +- .../org/polypheny/db/mqtt/StreamCapture.java | 147 +++++++++++++----- 2 files changed, 109 insertions(+), 44 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 00daa78931..2505da16f9 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -33,6 +33,7 @@ import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; +import org.polypheny.db.adapter.DataStore; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; @@ -251,9 +252,8 @@ void processMsg( Mqtt3Publish subMsg ) { // create Instance of PolyStream for the message with namespace //TODO: attention: return values, not correct, might need a change of type. String msg = StreamProcessing.processMsg( subMsg ); - StreamCapture streamCapture = new StreamCapture( this.transactionManager, this.namespace ); - streamCapture.saveMsgInDocument( subMsg.getTopic().toString(), msg ); - + //TODO: StreamCapture streamCapture = new StreamCapture( this.transactionManager, ); + //TODO: streamCapture.handleMessage( ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index ac808cdb7f..92becbb353 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -42,84 +42,145 @@ public class StreamCapture { TransactionManager transactionManager; PolyphenyHomeDirManager homeDirManager; - String namespace; + PolyStream stream; - StreamCapture( final TransactionManager transactionManager, String namespace ) { + StreamCapture( final TransactionManager transactionManager, PolyStream stream) { this.transactionManager = transactionManager; - this.namespace = namespace; + this.stream = stream; + } - public void saveMsgInDocument( String topic, String message ) { + public void handleContent( ) { //String path = registerTopicFolder(topic); - long schemaId = createCollection( topic ); - log.info( "schemaId = {}", schemaId ); - log.info( "testing method createCollection" ); - Catalog catalog = Catalog.getInstance(); - CatalogSchema schema = null; - schema = catalog.getSchema( schemaId ); + long storeId = getCollection( ); + if ( storeId != 0 ) { + stream.setStoreID( storeId ); + boolean saved = saveContent(); + log.info( "testing method createCollection" ); + Catalog catalog = Catalog.getInstance(); + CatalogSchema schema = null; + schema = catalog.getSchema( stream.getNamespaceID() ); - log.info( "Namespace name: {}, Namespace Type: {}, Database name: {}", schema.getName(), schema.getNamespaceType(), schema.getDatabaseName() ); - //TODO: create Document in collection with message - } - long createCollection( String topic ) { - return createCollection( topic, Catalog.defaultDatabaseId, Catalog.defaultUserId ); - } - long createCollection( String topic, long databaseId ) { - return createCollection( topic, databaseId, Catalog.defaultUserId ); - } - long createCollection( String topic, int userId ) { - return createCollection( topic, Catalog.defaultDatabaseId, userId ); - } + log.info( "Namespace name: {}, Namespace Type: {}, Database name: {}", schema.getName(), schema.getNamespaceType(), schema.getDatabaseName() ); - long createCollection( String topic, long databaseId, int userId ) { + } + } + /** + * + * @return the id of the collection that was already existing with the topic as name or that was newly created + */ + long getCollection( ) { + // TODO: schemaID ist NamespaceId ??? Catalog catalog = Catalog.getInstance(); // In case there is a namespace with the same name existing and is of Type Document, then that id of the existing schema will be used to create a collection. - long schemaId = schemaId = catalog.addNamespace( this.namespace, databaseId, userId, NamespaceType.DOCUMENT ); - - // check for namespace and NamespaceType: - try { - CatalogSchema schema = catalog.getSchema( databaseId, this.namespace ); + long schemaId = 0; + + // check for existing namespace with DOCUMENT NamespaceType: + if ( catalog.checkIfExistsSchema( stream.databaseId, stream.getNamespace() ) ) { + CatalogSchema schema = null; + try { + schema = catalog.getSchema( stream.databaseId, stream.getNamespace() ); + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + // TODO: what to do here? Maybe get schema again? + //Not this as Namespace already exists...OR create Namespace of Datatype Document ??? + // schemaId = catalog.addNamespace( this.namespace, databaseId, userId, NamespaceType.DOCUMENT ); + return 0; + } - if ( catalog.checkIfExistsSchema( databaseId, this.namespace ) && schema.namespaceType == NamespaceType.DOCUMENT ) { - schemaId = schema.id; + assert schema != null; + if (schema.namespaceType == NamespaceType.DOCUMENT ) { + //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic + List collectionList = catalog.getCollections( schema.id, null ); + for ( CatalogCollection collection : collectionList ) { + if( collection.name.equals( stream.topic ) ) { + //TODO: vll. Placement abfragen? + //collection.addPlacement( adapterId ); + //collection.removePlacement( adapterID ); + return collection.id; + // and return + } + } + + //TODO: no Collection with this name: so create Collection here, and return collection id + + + } else { + //TODO: Namespacetype is not of type Document -> what to do: + //maybe create new Namespace with type Document: + return createNewCollection(); } - } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); } + if ( addNewNamespace() ) { + return createNewCollection(); + } + return 0; + + } - //TODO: abfragen ob es collection schon gibt - //TODO: Namespace abfragen/erstellen und collection abfragen/erstellen in versch methoden machen + private boolean addNewNamespace( ) { + boolean methodFinished = false; + Catalog catalog = Catalog.getInstance(); + stream.setNamespaceID( catalog.addNamespace( stream.getNamespace(), stream.databaseId, stream.userId, NamespaceType.DOCUMENT ) ); + methodFinished = true; + return methodFinished; + } - Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; + + private long createNewCollection() { + // TODO: new collection + boolean methodFinished = false; + Catalog catalog = Catalog.getInstance(); + + //Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; Transaction transaction = null; Statement statement = null; try { - transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, databaseId, false, "MQTT Stream" ); + transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, stream.databaseId, false, "MQTT Stream" ); statement = transaction.createStatement(); } catch ( Exception e ) { log.error( "An error occurred: {}", e.getMessage() ); + return 0; } try { List dataStores = new ArrayList<>(); + //TODO: bei Placement adapter angeben! DdlManager.getInstance().createCollection( - schemaId, - topic, + this.stream.getNamespaceID(), + this.stream.topic, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, - placementType, + PlacementType.MANUAL, statement ); - log.info( "Created Collection with name: {}", topic ); + log.info( "Created Collection with name: {}", this.stream.topic ); + } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible due to: " + e.getMessage() ); + return 0; + } + + List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); + for ( int i = 0; i < collectionList.size(); i++ ) { + if( collectionList.get( i ).name.equals( this.stream.topic ) ) { + //TODO: vll. Placement einfügen? + int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameInterface() ).getAdapterId(); + collectionList.set( i, collectionList.get( i ).addPlacement( adapterId ) ); + + return collectionList.get( i ).id; + } } - return schemaId; + return 0; + } + boolean saveContent( ) { + //TODO: save Message here -> Polyalgebra + return true; } @@ -142,5 +203,9 @@ private static String registerTopicFolder( String topic ) { } + public static void main ( String[] args ) { + + } + } From d487f9106970b3396ea4e1a0135f050dbf91949e Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 2 Jul 2023 17:43:38 +0200 Subject: [PATCH 022/114] added instance variables added necessary instance variables for class StreamCapture. created few constructors. --- .../org/polypheny/db/mqtt/PolyStream.java | 69 +++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index 1cead079e9..702efc3a41 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -16,11 +16,72 @@ package org.polypheny.db.mqtt; +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; + public class PolyStream { - // Representation of 1 message in a namespace - String topic; - String message; - String namespace; + // Representation of 1 content in a namespace + @Getter + final String topic; + + @Getter + final String uniqueNameInterface; + @Getter + final String content; + @Getter + final String namespace; + @Getter + @Setter + long namespaceID; + @Getter + final long databaseId; + @Getter + final int userId; + @Getter + @Setter + long storeID; // the ID of collection/graph/table... the place where info is/should be saved + + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId ) { + this.topic = topic; + this.uniqueNameInterface = uniqueNameInterface; + this.content = content; + this.namespace = namespace; + this.databaseId = databaseId; + this.userId = userId; + } + + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId ) { + this.topic = topic; + this.uniqueNameInterface = uniqueNameInterface; + this.content = content; + this.namespace = namespace; + this.namespaceID = namespaceID; + this.databaseId = databaseId; + this.userId = userId; + } + + + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId, long storeID ) { + this.topic = topic; + this.uniqueNameInterface = uniqueNameInterface; + this.content = content; + this.namespace = namespace; + this.databaseId = databaseId; + this.userId = userId; + this.storeID = storeID; + } + + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId, long storeID ) { + this.topic = topic; + this.uniqueNameInterface = uniqueNameInterface; + this.content = content; + this.namespace = namespace; + this.namespaceID = namespaceID; + this.databaseId = databaseId; + this.userId = userId; + this.storeID = storeID; + } } From 440400b2f7211afa92a6d5673861034a8e82cdee Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 4 Jul 2023 18:08:56 +0200 Subject: [PATCH 023/114] started implementing saveContent - saveContent implementation not finished yet. - added Placements to createCollection --- .../org/polypheny/db/mqtt/StreamCapture.java | 205 +++++++++++++----- 1 file changed, 155 insertions(+), 50 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 92becbb353..2e0199847b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -18,22 +18,29 @@ import java.io.File; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.AdapterManager; import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.algebra.AlgNode; +import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.catalog.entity.CatalogSchema; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; -import org.polypheny.db.catalog.exceptions.NamespaceAlreadyExistsException; +import org.polypheny.db.catalog.exceptions.GenericCatalogException; +import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; +import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; +import org.polypheny.db.prepare.Context; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; import org.polypheny.db.util.PolyphenyHomeDirManager; @@ -45,16 +52,19 @@ public class StreamCapture { PolyStream stream; - StreamCapture( final TransactionManager transactionManager, PolyStream stream) { + StreamCapture( final TransactionManager transactionManager, PolyStream stream ) { this.transactionManager = transactionManager; this.stream = stream; - + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + this.stream.setUserId( statement.getPrepareContext().getCurrentUserId() ); + this.stream.setDatabaseId( statement.getPrepareContext().getDatabaseId() ); } - public void handleContent( ) { + public void handleContent() { //String path = registerTopicFolder(topic); - long storeId = getCollection( ); + long storeId = getCollection(); if ( storeId != 0 ) { stream.setStoreID( storeId ); boolean saved = saveContent(); @@ -68,11 +78,11 @@ public void handleContent( ) { } } + /** - * * @return the id of the collection that was already existing with the topic as name or that was newly created */ - long getCollection( ) { + long getCollection() { // TODO: schemaID ist NamespaceId ??? Catalog catalog = Catalog.getInstance(); @@ -80,10 +90,10 @@ long getCollection( ) { long schemaId = 0; // check for existing namespace with DOCUMENT NamespaceType: - if ( catalog.checkIfExistsSchema( stream.databaseId, stream.getNamespace() ) ) { + if ( catalog.checkIfExistsSchema( stream.getDatabaseId(), stream.getNamespace() ) ) { CatalogSchema schema = null; try { - schema = catalog.getSchema( stream.databaseId, stream.getNamespace() ); + schema = catalog.getSchema( stream.getDatabaseId(), stream.getNamespace() ); } catch ( UnknownSchemaException e ) { log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); // TODO: what to do here? Maybe get schema again? @@ -91,66 +101,68 @@ long getCollection( ) { // schemaId = catalog.addNamespace( this.namespace, databaseId, userId, NamespaceType.DOCUMENT ); return 0; } + //todo: rmv + log.info( "E Namespace with same name" ); assert schema != null; - if (schema.namespaceType == NamespaceType.DOCUMENT ) { + if ( schema.namespaceType == NamespaceType.DOCUMENT ) { + //todo: rmv + log.info( "E Namespace with type DOCUMENT" ); //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic - List collectionList = catalog.getCollections( schema.id, null ); + List collectionList = catalog.getCollections( schema.id, null ); for ( CatalogCollection collection : collectionList ) { - if( collection.name.equals( stream.topic ) ) { - //TODO: vll. Placement abfragen? - //collection.addPlacement( adapterId ); - //collection.removePlacement( adapterID ); - return collection.id; - // and return + if ( collection.name.equals( stream.topic ) ) { + //todo: rmv + log.info( "E Collection with topic as name" ); + int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameOfInterface() ).getAdapterId(); + if ( !collection.placements.contains( adapterId ) ) { + return collection.addPlacement( adapterId ).id; + } else { + return collection.id; + } } } - - //TODO: no Collection with this name: so create Collection here, and return collection id - + //todo: rmv + log.info( "No Collection with topic as name" ); + return createNewCollection(); } else { - //TODO: Namespacetype is not of type Document -> what to do: - //maybe create new Namespace with type Document: - return createNewCollection(); + //todo: rmv + log.info( "Namespace not of type DOCUMENT" ); + //TODO: Namespacetype is not of type Document -> IS this what you do: + if ( addNewNamespace() ) { + return createNewCollection(); + } + return 0; } - } - if ( addNewNamespace() ) { + } else if ( addNewNamespace() ) { + //todo: rmv + log.info( "No Namespace with choosen name" ); + log.info( "Created new Namespace" ); return createNewCollection(); } return 0; - } - private boolean addNewNamespace( ) { + + private boolean addNewNamespace() { boolean methodFinished = false; Catalog catalog = Catalog.getInstance(); - stream.setNamespaceID( catalog.addNamespace( stream.getNamespace(), stream.databaseId, stream.userId, NamespaceType.DOCUMENT ) ); + stream.setNamespaceID( catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ) ); methodFinished = true; return methodFinished; } private long createNewCollection() { - // TODO: new collection - boolean methodFinished = false; Catalog catalog = Catalog.getInstance(); //Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; - Transaction transaction = null; - Statement statement = null; - - try { - transaction = this.transactionManager.startTransaction( Catalog.defaultUserId, stream.databaseId, false, "MQTT Stream" ); - statement = transaction.createStatement(); - } catch ( Exception e ) { - log.error( "An error occurred: {}", e.getMessage() ); - return 0; - } + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); try { List dataStores = new ArrayList<>(); - //TODO: bei Placement adapter angeben! DdlManager.getInstance().createCollection( this.stream.getNamespaceID(), this.stream.topic, @@ -164,12 +176,11 @@ private long createNewCollection() { log.error( "The generation of the collection was not possible due to: " + e.getMessage() ); return 0; } - - List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); + //add placement + List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); for ( int i = 0; i < collectionList.size(); i++ ) { - if( collectionList.get( i ).name.equals( this.stream.topic ) ) { - //TODO: vll. Placement einfügen? - int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameInterface() ).getAdapterId(); + if ( collectionList.get( i ).name.equals( this.stream.topic ) ) { + int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameOfInterface() ).getAdapterId(); collectionList.set( i, collectionList.get( i ).addPlacement( adapterId ) ); return collectionList.get( i ).id; @@ -178,12 +189,105 @@ private long createNewCollection() { return 0; } - boolean saveContent( ) { - //TODO: save Message here -> Polyalgebra + + boolean saveContent() { +/** + //TODO: save Message here -> Polyalgebra + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + AlgBuilder algBuilder = AlgBuilder.create( statement ); + JavaTypeFactory typeFactory = transaction.getTypeFactory(); + RexBuilder rexBuilder = new RexBuilder( typeFactory ); + + PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); + List names = new ArrayList<>(); + //TODO: change naming maybe + names.add( this.stream.topic ); + AlgOptTable table = catalogReader.getCollection( names ); + + // Values + AlgDataType tableRowType = table.getRowType( ); + List tableRows = tableRowType.getFieldList(); + + AlgOptPlanner planner = statement.getQueryProcessor().getPlanner(); + AlgOptCluster cluster = AlgOptCluster.create( planner, rexBuilder ); + + List valueColumnNames = this.valuesColumnNames( insertValueRequest.values ); + List rexValues = this.valuesNode( statement, algBuilder, rexBuilder, insertValueRequest, tableRows, inputStreams ).get( 0 ); + algBuilder.push( LogicalValues.createOneRow( cluster ) ); + algBuilder.project( rexValues, valueColumnNames ); + + // Table Modify + AlgNode algNode = algBuilder.build(); + Modify modify = new LogicalModify( + cluster, + algNode.getTraitSet(), + table, + catalogReader, + algNode, + LogicalModify.Operation.INSERT, + null, + null, + false + ); + + // Wrap {@link AlgNode} into a RelRoot + final AlgDataType rowType = modify.getRowType(); + final List> fields = Pair.zip( ImmutableIntList.identity( rowType.getFieldCount() ), rowType.getFieldNames() ); + final AlgCollation collation = + algNode instanceof Sort + ? ((Sort) algNode).collation + : AlgCollations.EMPTY; + AlgRoot root = new AlgRoot( modify, rowType, Kind.INSERT, fields, collation ); + log.debug( "AlgRoot was built." ); + + Context ctx = statement.getPrepareContext(); + log.info( executeAndTransformPolyAlg( root, statement, ctx ) ); + **/ return true; } + private Transaction getTransaction() { + try { + return transactionManager.startTransaction( this.stream.getUserId(), this.stream.getDatabaseId(), false, "MQTT Stream" ); + } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { + throw new RuntimeException( "Error while starting transaction", e ); + } + } + + + String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { + //TODO: implement + + try { + // Prepare + PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); + log.debug( "AlgRoot was prepared." ); + + final Iterable iterable = result.enumerable( statement.getDataContext() ); + Iterator iterator = iterable.iterator(); + while ( iterator.hasNext() ) { + iterator.next(); + } + + statement.getTransaction().commit(); + } catch ( Throwable e ) { + log.error( "Error during execution of REST query", e ); + try { + statement.getTransaction().rollback(); + } catch ( TransactionException transactionException ) { + log.error( "Could not rollback", e ); + } + return null; + } + //Pair result = restResult.getResult( ctx ); + + //return result.left; + return null; + } + + private static String registerTopicFolder( String topic ) { PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); @@ -203,7 +307,8 @@ private static String registerTopicFolder( String topic ) { } - public static void main ( String[] args ) { + + public static void main( String[] args ) { } From 85c96e02ad84ac531842c5c276f7e77d84dc5313 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 4 Jul 2023 18:09:55 +0200 Subject: [PATCH 024/114] added PolyStream and connected processMsg with StreamCapture class --- .../java/org/polypheny/db/mqtt/MqttStreamPlugin.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 2505da16f9..d7f3b062bc 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -33,7 +33,6 @@ import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; -import org.polypheny.db.adapter.DataStore; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; @@ -249,11 +248,11 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { - // create Instance of PolyStream for the message with namespace //TODO: attention: return values, not correct, might need a change of type. - String msg = StreamProcessing.processMsg( subMsg ); - //TODO: StreamCapture streamCapture = new StreamCapture( this.transactionManager, ); - //TODO: streamCapture.handleMessage( ); + String content = StreamProcessing.processMsg( subMsg ); + PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace ); + StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream); + streamCapture.handleContent( ); } From d6dd1bff757d2f0fd6ca5e47a4c00c93c9686a0d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 4 Jul 2023 18:11:09 +0200 Subject: [PATCH 025/114] made changes in modifiers and new constructor --- .../org/polypheny/db/mqtt/PolyStream.java | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index 702efc3a41..d8529d8650 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -17,7 +17,6 @@ package org.polypheny.db.mqtt; import lombok.Getter; -import lombok.NonNull; import lombok.Setter; public class PolyStream { @@ -27,34 +26,46 @@ public class PolyStream { final String topic; @Getter - final String uniqueNameInterface; + final String uniqueNameOfInterface; @Getter final String content; @Getter final String namespace; @Getter @Setter - long namespaceID; + private long namespaceID; + @Setter @Getter - final long databaseId; + private long databaseId; + @Setter @Getter - final int userId; + private int userId; @Getter @Setter - long storeID; // the ID of collection/graph/table... the place where info is/should be saved + private long storeID; // the ID of collection/graph/table... the place where info is/should be saved + + + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace ) { + this.topic = topic; + this.uniqueNameOfInterface = uniqueNameInterface; + this.content = content; + this.namespace = namespace; + } + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId ) { this.topic = topic; - this.uniqueNameInterface = uniqueNameInterface; + this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; this.databaseId = databaseId; this.userId = userId; } + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId ) { this.topic = topic; - this.uniqueNameInterface = uniqueNameInterface; + this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; this.namespaceID = namespaceID; @@ -65,7 +76,7 @@ public PolyStream( String topic, String uniqueNameInterface, String content, Str public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId, long storeID ) { this.topic = topic; - this.uniqueNameInterface = uniqueNameInterface; + this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; this.databaseId = databaseId; @@ -73,9 +84,10 @@ public PolyStream( String topic, String uniqueNameInterface, String content, Str this.storeID = storeID; } + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId, long storeID ) { this.topic = topic; - this.uniqueNameInterface = uniqueNameInterface; + this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; this.namespaceID = namespaceID; From b118f390cdd6155aa39ad75f244c36cd4b8b7765 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 7 Jul 2023 09:48:10 +0200 Subject: [PATCH 026/114] resolved problem of changes in the settings not being permanently saved --- .../org/polypheny/db/catalog/Catalog.java | 8 +++ .../polypheny/db/iface/QueryInterface.java | 64 +++++++++++++++++++ .../org/polypheny/db/catalog/MockCatalog.java | 6 ++ .../org/polypheny/db/catalog/CatalogImpl.java | 16 +++++ 4 files changed, 94 insertions(+) diff --git a/core/src/main/java/org/polypheny/db/catalog/Catalog.java b/core/src/main/java/org/polypheny/db/catalog/Catalog.java index 644086c921..3bb729294f 100644 --- a/core/src/main/java/org/polypheny/db/catalog/Catalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/Catalog.java @@ -1233,6 +1233,14 @@ protected final boolean isValidIdentifier( final String str ) { */ public abstract void deleteQueryInterface( int ifaceId ); + /** + * Update settings of a query interface + * + * @param queryInterfaceId The id of the query interface + * @param newSettings The new settings for the query interface + */ + public abstract void updateQueryInterfaceSettings( int queryInterfaceId, Map newSettings ); + /** * Adds a partition to the catalog * diff --git a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java index 0f56938101..b16883849a 100644 --- a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java +++ b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java @@ -17,6 +17,9 @@ package org.polypheny.db.iface; +import static org.reflections.Reflections.log; + +import com.google.common.collect.ImmutableList; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.*; @@ -25,7 +28,21 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.pf4j.ExtensionPoint; +import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.PlacementType; +import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.GenericCatalogException; +import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaException; +import org.polypheny.db.catalog.exceptions.UnknownUserException; +import org.polypheny.db.ddl.DdlManager; +import org.polypheny.db.ddl.DdlManager.ConstraintInformation; +import org.polypheny.db.ddl.DdlManager.FieldInformation; import org.polypheny.db.languages.LanguageManager; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; @@ -122,6 +139,53 @@ protected void validateSettings( Map newSettings, boolean initia public void updateSettings( Map newSettings ) { this.validateSettings( newSettings, false ); List updatedSettings = this.applySettings( newSettings ); + Catalog catalog = Catalog.getInstance(); + catalog.updateQueryInterfaceSettings(getQueryInterfaceId(), getCurrentSettings()); + + Transaction transaction = null; + try { + transaction = transactionManager.startTransaction( Catalog.defaultUserId, Catalog.defaultDatabaseId, false, catalog.getQueryInterface( queryInterfaceId ).name); + } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { + throw new RuntimeException( "Error while starting transaction", e ); + } + + Statement statement = transaction.createStatement(); + long schemaId = 0; + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, "SettingsSchema"); + schemaId = schema.id; + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + } + + List stores = null; + + PlacementType placementType = PlacementType.AUTOMATIC; + + List columns = null; + + List constraints = null; + try{ + DdlManager.getInstance().createTable( + schemaId, + "tableName", + columns, + constraints, + true, + stores, + placementType, + statement ); + + } catch (Exception e) { + log.error( "Could not create table to commit the new settings." ); + } + + try{ + transaction.commit(); + } catch ( TransactionException e ) { + log.error( "Could not commit new changes in settings." ); + } this.reloadSettings( updatedSettings ); } diff --git a/core/src/test/java/org/polypheny/db/catalog/MockCatalog.java b/core/src/test/java/org/polypheny/db/catalog/MockCatalog.java index 885ffae660..e44764358a 100644 --- a/core/src/test/java/org/polypheny/db/catalog/MockCatalog.java +++ b/core/src/test/java/org/polypheny/db/catalog/MockCatalog.java @@ -893,6 +893,12 @@ public void deleteQueryInterface( int ifaceId ) { } + @Override + public void updateQueryInterfaceSettings( int queryInterfaceId, Map newSettings ) { + throw new NotImplementedException(); + } + + @Override public long addPartitionGroup( long tableId, String partitionGroupName, long schemaId, PartitionType partitionType, long numberOfInternalPartitions, List effectivePartitionGroupQualifier, boolean isUnbound ) throws GenericCatalogException { throw new NotImplementedException(); diff --git a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java index 47d23dc131..509e2df9d5 100644 --- a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java +++ b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java @@ -4014,6 +4014,22 @@ public void deleteQueryInterface( int ifaceId ) { } } + /** + * {@inheritDoc} + */ + public void updateQueryInterfaceSettings( int queryInterfaceId, Map newSettings ) { + CatalogQueryInterface old = getQueryInterface( queryInterfaceId ); + Map temp = new HashMap<>(); + newSettings.forEach( temp::put ); + temp.values().forEach( System.out::println ); + CatalogQueryInterface queryInterface = new CatalogQueryInterface( old.id, old.name, old.clazz, temp ); + synchronized ( this ) { + queryInterfaces.put( queryInterface.id, queryInterface ); + queryInterfaceNames.put( queryInterface.name, queryInterface ); + } + listeners.firePropertyChange( "queryInterface", old, queryInterface ); + } + /** * {@inheritDoc} From d7f48aa737a9f7fbf5a512b44e174bde059a4af1 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Fri, 7 Jul 2023 10:17:36 +0200 Subject: [PATCH 027/114] Improve update settings fix --- .../org/polypheny/db/adapter/Adapter.java | 9 ++- .../polypheny/db/iface/QueryInterface.java | 74 +++---------------- .../org/polypheny/db/catalog/CatalogImpl.java | 1 - 3 files changed, 19 insertions(+), 65 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/adapter/Adapter.java b/core/src/main/java/org/polypheny/db/adapter/Adapter.java index d5b97e93fb..f562859d80 100644 --- a/core/src/main/java/org/polypheny/db/adapter/Adapter.java +++ b/core/src/main/java/org/polypheny/db/adapter/Adapter.java @@ -53,6 +53,7 @@ import org.polypheny.db.catalog.entity.CatalogColumnPlacement; import org.polypheny.db.catalog.entity.CatalogPartitionPlacement; import org.polypheny.db.catalog.entity.CatalogTable; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.config.Config; import org.polypheny.db.config.Config.ConfigListener; import org.polypheny.db.config.ConfigDocker; @@ -406,7 +407,13 @@ public void updateSettings( Map newSettings ) { this.validateSettings( newSettings, false ); List updatedSettings = this.applySettings( newSettings ); this.reloadSettings( updatedSettings ); - Catalog.getInstance().updateAdapterSettings( getAdapterId(), newSettings ); + Catalog catalog = Catalog.getInstance(); + catalog.updateAdapterSettings( getAdapterId(), getCurrentSettings() ); + try { + catalog.commit(); + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException( e ); + } } diff --git a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java index b16883849a..c0e5ee1c5d 100644 --- a/core/src/main/java/org/polypheny/db/iface/QueryInterface.java +++ b/core/src/main/java/org/polypheny/db/iface/QueryInterface.java @@ -17,32 +17,20 @@ package org.polypheny.db.iface; -import static org.reflections.Reflections.log; - -import com.google.common.collect.ImmutableList; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; - +import java.util.Objects; import lombok.AllArgsConstructor; import lombok.Getter; import org.pf4j.ExtensionPoint; -import org.polypheny.db.adapter.DataStore; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PlacementType; -import org.polypheny.db.catalog.entity.CatalogSchema; -import org.polypheny.db.catalog.exceptions.GenericCatalogException; -import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; -import org.polypheny.db.catalog.exceptions.UnknownSchemaException; -import org.polypheny.db.catalog.exceptions.UnknownUserException; -import org.polypheny.db.ddl.DdlManager; -import org.polypheny.db.ddl.DdlManager.ConstraintInformation; -import org.polypheny.db.ddl.DdlManager.FieldInformation; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.languages.LanguageManager; -import org.polypheny.db.transaction.Statement; -import org.polypheny.db.transaction.Transaction; -import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; @@ -139,54 +127,14 @@ protected void validateSettings( Map newSettings, boolean initia public void updateSettings( Map newSettings ) { this.validateSettings( newSettings, false ); List updatedSettings = this.applySettings( newSettings ); + this.reloadSettings( updatedSettings ); Catalog catalog = Catalog.getInstance(); - catalog.updateQueryInterfaceSettings(getQueryInterfaceId(), getCurrentSettings()); - - Transaction transaction = null; + Catalog.getInstance().updateQueryInterfaceSettings( getQueryInterfaceId(), getCurrentSettings() ); try { - transaction = transactionManager.startTransaction( Catalog.defaultUserId, Catalog.defaultDatabaseId, false, catalog.getQueryInterface( queryInterfaceId ).name); - } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { - throw new RuntimeException( "Error while starting transaction", e ); - } - - Statement statement = transaction.createStatement(); - long schemaId = 0; - CatalogSchema schema = null; - try { - schema = catalog.getSchema( Catalog.defaultDatabaseId, "SettingsSchema"); - schemaId = schema.id; - } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - } - - List stores = null; - - PlacementType placementType = PlacementType.AUTOMATIC; - - List columns = null; - - List constraints = null; - try{ - DdlManager.getInstance().createTable( - schemaId, - "tableName", - columns, - constraints, - true, - stores, - placementType, - statement ); - - } catch (Exception e) { - log.error( "Could not create table to commit the new settings." ); - } - - try{ - transaction.commit(); - } catch ( TransactionException e ) { - log.error( "Could not commit new changes in settings." ); + catalog.commit(); + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException( e ); } - this.reloadSettings( updatedSettings ); } diff --git a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java index 509e2df9d5..275d43e49a 100644 --- a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java +++ b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java @@ -4021,7 +4021,6 @@ public void updateQueryInterfaceSettings( int queryInterfaceId, Map temp = new HashMap<>(); newSettings.forEach( temp::put ); - temp.values().forEach( System.out::println ); CatalogQueryInterface queryInterface = new CatalogQueryInterface( old.id, old.name, old.clazz, temp ); synchronized ( this ) { queryInterfaces.put( queryInterface.id, queryInterface ); From e165e3764fd214fa8b20d3c7ec7d6416df573517 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 7 Jul 2023 15:46:02 +0200 Subject: [PATCH 028/114] inserted option of using default databaseID and userID --- .../src/main/java/org/polypheny/db/mqtt/PolyStream.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index d8529d8650..fb0c7ee69e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -18,6 +18,7 @@ import lombok.Getter; import lombok.Setter; +import org.polypheny.db.catalog.Catalog; public class PolyStream { @@ -50,6 +51,8 @@ public PolyStream( String topic, String uniqueNameInterface, String content, Str this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; + this.databaseId = Catalog.defaultDatabaseId; + this.userId = Catalog.defaultUserId; } From 4b8e37a353b454fed26c71f071cb039d0bab9a38 Mon Sep 17 00:00:00 2001 From: datomo Date: Fri, 7 Jul 2023 16:50:49 +0200 Subject: [PATCH 029/114] added example for doc insert and scan, added dedicate methods in AlgBuilder --- .../org/polypheny/db/tools/AlgBuilder.java | 139 ++++++++++++++++-- plugins/mqtt-stream/build.gradle | 3 + .../org/polypheny/db/mqtt/StreamCapture.java | 58 +++++++- 3 files changed, 182 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java b/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java index 87a5bf4ec8..7a43dd1fac 100644 --- a/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java +++ b/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java @@ -34,20 +34,67 @@ package org.polypheny.db.tools; +import static org.polypheny.db.util.Static.RESOURCE; + import com.google.common.base.Preconditions; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import java.math.BigDecimal; +import java.util.AbstractList; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Experimental; +import org.bson.BsonDocument; +import org.bson.BsonElement; import org.bson.BsonValue; -import org.polypheny.db.algebra.*; +import org.polypheny.db.algebra.AlgCollation; +import org.polypheny.db.algebra.AlgCollations; +import org.polypheny.db.algebra.AlgDistribution; +import org.polypheny.db.algebra.AlgFieldCollation; +import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.algebra.constant.SemiJoinType; -import org.polypheny.db.algebra.core.*; +import org.polypheny.db.algebra.core.Aggregate; +import org.polypheny.db.algebra.core.AggregateCall; +import org.polypheny.db.algebra.core.AlgFactories; import org.polypheny.db.algebra.core.AlgFactories.ScanFactory; +import org.polypheny.db.algebra.core.CorrelationId; +import org.polypheny.db.algebra.core.Filter; +import org.polypheny.db.algebra.core.Intersect; +import org.polypheny.db.algebra.core.Join; +import org.polypheny.db.algebra.core.JoinAlgType; +import org.polypheny.db.algebra.core.Match; +import org.polypheny.db.algebra.core.Minus; +import org.polypheny.db.algebra.core.Modify.Operation; +import org.polypheny.db.algebra.core.Project; +import org.polypheny.db.algebra.core.Scan; +import org.polypheny.db.algebra.core.SemiJoin; +import org.polypheny.db.algebra.core.Sort; +import org.polypheny.db.algebra.core.Union; +import org.polypheny.db.algebra.core.Values; import org.polypheny.db.algebra.fun.AggFunction; +import org.polypheny.db.algebra.logical.document.LogicalDocumentModify; import org.polypheny.db.algebra.logical.document.LogicalDocumentProject; import org.polypheny.db.algebra.logical.document.LogicalDocumentScan; +import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; import org.polypheny.db.algebra.logical.lpg.LogicalGraph; import org.polypheny.db.algebra.logical.lpg.LogicalLpgMatch; import org.polypheny.db.algebra.logical.lpg.LogicalLpgProject; @@ -63,8 +110,24 @@ import org.polypheny.db.languages.OperatorRegistry; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.nodes.Operator; -import org.polypheny.db.plan.*; -import org.polypheny.db.rex.*; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgOptPredicateList; +import org.polypheny.db.plan.AlgOptSchema; +import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.plan.AlgOptUtil; +import org.polypheny.db.plan.Context; +import org.polypheny.db.plan.Contexts; +import org.polypheny.db.prepare.PolyphenyDbCatalogReader; +import org.polypheny.db.rex.RexBuilder; +import org.polypheny.db.rex.RexCall; +import org.polypheny.db.rex.RexCorrelVariable; +import org.polypheny.db.rex.RexExecutor; +import org.polypheny.db.rex.RexInputRef; +import org.polypheny.db.rex.RexLiteral; +import org.polypheny.db.rex.RexNode; +import org.polypheny.db.rex.RexShuttle; +import org.polypheny.db.rex.RexSimplify; +import org.polypheny.db.rex.RexUtil; import org.polypheny.db.runtime.Hook; import org.polypheny.db.runtime.PolyCollections.PolyDictionary; import org.polypheny.db.schema.ModelTrait; @@ -72,17 +135,21 @@ import org.polypheny.db.schema.graph.PolyNode; import org.polypheny.db.transaction.Statement; import org.polypheny.db.type.PolyType; -import org.polypheny.db.util.*; +import org.polypheny.db.util.DateString; +import org.polypheny.db.util.Holder; +import org.polypheny.db.util.ImmutableBitSet; +import org.polypheny.db.util.ImmutableIntList; +import org.polypheny.db.util.ImmutableNullableList; +import org.polypheny.db.util.Litmus; +import org.polypheny.db.util.NlsString; +import org.polypheny.db.util.Pair; +import org.polypheny.db.util.TimeString; +import org.polypheny.db.util.TimestampString; +import org.polypheny.db.util.Util; +import org.polypheny.db.util.ValidatorUtil; import org.polypheny.db.util.mapping.Mapping; import org.polypheny.db.util.mapping.Mappings; -import javax.annotation.Nonnull; -import java.math.BigDecimal; -import java.util.*; -import java.util.stream.Collectors; - -import static org.polypheny.db.util.Static.RESOURCE; - /** * Builder for relational expressions. @@ -2406,7 +2473,51 @@ public AlgBuilder permute( Mapping mapping ) { public AlgBuilder aggregate( GroupKey groupKey, List aggregateCalls ) { - return aggregate( groupKey, Lists.transform( aggregateCalls, AggCallImpl2::new ) ); + return aggregate( groupKey, aggregateCalls.stream().map( AggCallImpl2::new ).collect( Collectors.toList() ) ); + } + + // DOCUMENT SCAN + + + public AlgBuilder docScan( Statement statement, String collectionName ) { + PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); + AlgOptTable collection = catalogReader.getCollection( List.of( collectionName.split( "\\." ) ) ); + stack.push( new Frame( LogicalDocumentScan.create( cluster, collection ) ) ); + return this; + } + + + // DOCUMENT MODIFY + public AlgBuilder docInsert( Statement statement, String collectionName, BsonDocument... documents ) { + PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); + AlgOptTable collection = catalogReader.getCollection( List.of( collectionName.split( "\\." ) ) ); + + AlgNode values = LogicalDocumentValues.create( cluster, ImmutableList.copyOf( documents ) ); + + LogicalDocumentModify modify = LogicalDocumentModify.create( collection, values, catalogReader, Operation.INSERT, null, null ); + + stack.push( new Frame( modify ) ); + return this; + } + + + public AlgBuilder docInsert( Statement statement, String collectionName, BsonElement... elements ) { + return docInsert( statement, collectionName, new BsonDocument( List.of( elements ) ) ); + } + + + public AlgBuilder docInsert( Statement statement, String collectionName, String key, BsonValue value ) { + return docInsert( statement, collectionName, new BsonDocument( key, value ) ); + } + + + public AlgBuilder docInsert( Statement statement, String collectionName, String key0, BsonValue value0, String key1, BsonValue value1 ) { + return docInsert( statement, collectionName, new BsonElement( key0, value0 ), new BsonElement( key1, value1 ) ); + } + + + public AlgBuilder docInsert( Statement statement, String collectionName, String key0, BsonValue value0, String key1, BsonValue value1, String key2, BsonValue value2 ) { + return docInsert( statement, collectionName, new BsonElement( key0, value0 ), new BsonElement( key1, value1 ), new BsonElement( key2, value2 ) ); } diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 2286f228c0..c0fd28c93c 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -4,10 +4,13 @@ group "org.polypheny" dependencies { compileOnly project(":core") compileOnly project(":monitoring") + compileOnly project(":plugins:mql-language") // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: hivemq_mqttclient_version + implementation group: "org.mongodb", name: "mongodb-driver-sync", version: mongodb_driver_sync_version // Apache 2.0 + // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") testImplementation project(path: ":core") diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 2e0199847b..96d528a1aa 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -18,14 +18,17 @@ import java.io.File; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.bson.BsonDocument; +import org.bson.BsonInt32; +import org.bson.BsonString; import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.AdapterManager; import org.polypheny.db.adapter.DataStore; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; @@ -38,6 +41,7 @@ import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.prepare.Context; +import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; @@ -190,6 +194,49 @@ private long createNewCollection() { } + // added by Datomo + public void insertDocument() { + String collectionName = "users"; + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + + // Builder which allows to construct the algebra tree which is equivalent to query and is executed + AlgBuilder builder = AlgBuilder.create( statement ); + + // we insert document { age: 28, name: "David" } into the collection users + BsonDocument document = new BsonDocument(); + document.put( "age", new BsonInt32( 28 ) ); + document.put( "name", new BsonString( "David" ) ); + + AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); + + // we can then wrap the tree in an AlgRoot and execute it + AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); + // for inserts and all DML queries only a number is returned + String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + + + } + + + // added by Datomo + public void scanDocument() { + String collectionName = "users"; + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + + // Builder which allows to construct the algebra tree which is equivalent to query and is executed + AlgBuilder builder = AlgBuilder.create( statement ); + + AlgNode algNode = builder.docScan( statement, collectionName ).build(); + + // we can then wrap the tree in an AlgRoot and execute it + AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); + String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + + } + + boolean saveContent() { /** //TODO: save Message here -> Polyalgebra @@ -265,13 +312,16 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); log.debug( "AlgRoot was prepared." ); - final Iterable iterable = result.enumerable( statement.getDataContext() ); + /*final Iterable iterable = result.enumerable( statement.getDataContext() ); Iterator iterator = iterable.iterator(); while ( iterator.hasNext() ) { iterator.next(); - } + }*/ + // todo transform into desired output format + List> rows = result.getRows( statement, -1 ); statement.getTransaction().commit(); + return rows.toString(); } catch ( Throwable e ) { log.error( "Error during execution of REST query", e ); try { @@ -284,7 +334,7 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C //Pair result = restResult.getResult( ctx ); //return result.left; - return null; + //return null; } From 62b2553030a590398f1ddc1015be12b4ce70e82d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 7 Jul 2023 17:53:57 +0200 Subject: [PATCH 030/114] debugged namespace and collection creation logic methods: getCollection, createNamespace, createNewCollection --- .../org/polypheny/db/mqtt/StreamCapture.java | 78 +++++++------------ 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 2e0199847b..8463999c4b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -22,7 +22,6 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.PolyImplementation; -import org.polypheny.db.adapter.AdapterManager; import org.polypheny.db.adapter.DataStore; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; @@ -33,10 +32,12 @@ import org.polypheny.db.catalog.entity.CatalogSchema; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; import org.polypheny.db.catalog.exceptions.GenericCatalogException; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; +import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.prepare.Context; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; @@ -64,93 +65,69 @@ public class StreamCapture { public void handleContent() { //String path = registerTopicFolder(topic); - long storeId = getCollection(); + long storeId = getCollectionID(); if ( storeId != 0 ) { stream.setStoreID( storeId ); boolean saved = saveContent(); - log.info( "testing method createCollection" ); - Catalog catalog = Catalog.getInstance(); - CatalogSchema schema = null; - schema = catalog.getSchema( stream.getNamespaceID() ); - - log.info( "Namespace name: {}, Namespace Type: {}, Database name: {}", schema.getName(), schema.getNamespaceType(), schema.getDatabaseName() ); - + //TODO: gescheite Tests +// Catalog catalog = Catalog.getInstance(); +// CatalogSchema schema = null; +// schema = catalog.getSchema( stream.getNamespaceID() ); } } /** - * @return the id of the collection that was already existing with the topic as name or that was newly created + * @return the id of the collection that was either already existing with the topic as name or that was newly created */ - long getCollection() { - // TODO: schemaID ist NamespaceId ??? + long getCollectionID() { Catalog catalog = Catalog.getInstance(); + List schemaList = catalog.getSchemas( this.stream.getDatabaseId(), null ); - // In case there is a namespace with the same name existing and is of Type Document, then that id of the existing schema will be used to create a collection. - long schemaId = 0; // check for existing namespace with DOCUMENT NamespaceType: if ( catalog.checkIfExistsSchema( stream.getDatabaseId(), stream.getNamespace() ) ) { + CatalogSchema schema = null; try { schema = catalog.getSchema( stream.getDatabaseId(), stream.getNamespace() ); } catch ( UnknownSchemaException e ) { log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - // TODO: what to do here? Maybe get schema again? - //Not this as Namespace already exists...OR create Namespace of Datatype Document ??? - // schemaId = catalog.addNamespace( this.namespace, databaseId, userId, NamespaceType.DOCUMENT ); return 0; } - //todo: rmv - log.info( "E Namespace with same name" ); assert schema != null; if ( schema.namespaceType == NamespaceType.DOCUMENT ) { - //todo: rmv - log.info( "E Namespace with type DOCUMENT" ); + this.stream.setNamespaceID( schema.id ); //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic List collectionList = catalog.getCollections( schema.id, null ); for ( CatalogCollection collection : collectionList ) { if ( collection.name.equals( stream.topic ) ) { - //todo: rmv - log.info( "E Collection with topic as name" ); - int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameOfInterface() ).getAdapterId(); - if ( !collection.placements.contains( adapterId ) ) { - return collection.addPlacement( adapterId ).id; + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.uniqueNameOfInterface ).getQueryInterfaceId(); + if ( !collection.placements.contains( queryInterfaceId ) ) { + return collection.addPlacement( queryInterfaceId ).id; } else { return collection.id; } } } - //todo: rmv - log.info( "No Collection with topic as name" ); return createNewCollection(); } else { - //todo: rmv - log.info( "Namespace not of type DOCUMENT" ); - //TODO: Namespacetype is not of type Document -> IS this what you do: - if ( addNewNamespace() ) { - return createNewCollection(); - } - return 0; + this.stream.setNamespaceID( createNamespace() ); + return createNewCollection(); } - } else if ( addNewNamespace() ) { - //todo: rmv - log.info( "No Namespace with choosen name" ); - log.info( "Created new Namespace" ); + } else { + this.stream.setNamespaceID( createNamespace() ); return createNewCollection(); } - return 0; } - private boolean addNewNamespace() { - boolean methodFinished = false; + private long createNamespace() { Catalog catalog = Catalog.getInstance(); - stream.setNamespaceID( catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ) ); - methodFinished = true; - return methodFinished; + //TOdO: commit + return catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); } @@ -171,17 +148,20 @@ private long createNewCollection() { PlacementType.MANUAL, statement ); log.info( "Created Collection with name: {}", this.stream.topic ); - + transaction.commit(); } catch ( EntityAlreadyExistsException e ) { - log.error( "The generation of the collection was not possible due to: " + e.getMessage() ); + log.error( "The generation of the collection was not possible because there is a collaction already existing with this name." ); + return 0; + } catch ( TransactionException e ) { + log.error( "The commit after creating a new Collection could be completed!" ); return 0; } //add placement List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); for ( int i = 0; i < collectionList.size(); i++ ) { if ( collectionList.get( i ).name.equals( this.stream.topic ) ) { - int adapterId = AdapterManager.getInstance().getStore( this.stream.getUniqueNameOfInterface() ).getAdapterId(); - collectionList.set( i, collectionList.get( i ).addPlacement( adapterId ) ); + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + collectionList.set( i, collectionList.get( i ).addPlacement( queryInterfaceId ) ); return collectionList.get( i ).id; } From 2d46042379fe9bef0730a7d9bd117901e881d0ae Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 7 Jul 2023 17:54:45 +0200 Subject: [PATCH 031/114] improved trimming of topics coming from UI --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index d7f3b062bc..1bdf1e6de5 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -134,8 +134,11 @@ public void run() { log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - - subscribe( Arrays.asList( settings.get( "topics" ).trim().split( "," ) ) ); + /**List topicsList = new ArrayList<>( List.of( this.settings.get( "topics" ).split( "," ) ) ); + for ( int i = 0; i < topicsList.size(); i++ ) { + topicsList.set( i, topicsList.get(i).trim() ); + }**/ + subscribe( topicsToList( this.settings.get( "topics" ) ) ); } } ); @@ -165,12 +168,20 @@ public void shutdown() { } + public List topicsToList( String topics ) { + List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); + for ( int i = 0; i < topicsList.size(); i++ ) { + topicsList.set( i, topicsList.get(i).trim() ); + } + return topicsList; + } + @Override protected void reloadSettings( List updatedSettings ) { if ( updatedSettings.contains( "topics" ) ) { - List newTopicsList = Stream.of( this.getCurrentSettings().get( "topics" ).split( "," ) ).map( String::trim ).collect( Collectors.toList() ); + List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { @@ -219,7 +230,7 @@ public void subscribe( String topic ) { log.info( "Subscription was not successfull. Please try again." ); } else { this.topics.add( topic ); - log.info( "Successful subscription to topic {}.", topic ); + log.info( "Successful subscription to topic:{}.", topic ); } } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). @@ -241,7 +252,7 @@ public void unsubscribe( String topic ) { log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { this.topics.remove( topic ); - log.info( String.format( "Unsubscribed from %s.", topic ) ); + log.info( "Unsubscribed from topic:{}.", topic ); } } ); } From 8a2801d0bb03558868eef73883dbbd7bde34ba8b Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 9 Jul 2023 10:00:09 +0200 Subject: [PATCH 032/114] added namespace and type handling Now namespace will be checked in MqttStreamServer --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 130 ++++++++++++------ .../org/polypheny/db/mqtt/PolyStream.java | 56 ++------ 2 files changed, 98 insertions(+), 88 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 1bdf1e6de5..9746bf240a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -22,17 +22,17 @@ import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; @@ -61,6 +61,9 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); + // TODO namespace und type müssen dringend eingegeben werden, so machen wie bei topics? + mqttDefaultSettings.put( "namespace", "public" ); + mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -83,7 +86,8 @@ public static class MqttStreamServer extends QueryInterface { public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), - new QueryInterfaceSettingString( "namespace", false, true, false, null ), + new QueryInterfaceSettingString( "namespace", false, true, true, null ), + new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "RELATIONAL", "DOCUMENT", "GRAPH" ) ) ), new QueryInterfaceSettingString( "topics", false, true, true, null ) ); @@ -93,10 +97,11 @@ public static class MqttStreamServer extends QueryInterface { private ArrayList topics = new ArrayList(); - public static Mqtt3AsyncClient client; + private static Mqtt3AsyncClient client; - @Getter - public final String namespace; + private String namespace; + + private NamespaceType namespaceType; private final MonitoringPage monitoringPage; @@ -104,10 +109,15 @@ public static class MqttStreamServer extends QueryInterface { public MqttStreamServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); // Add information page - monitoringPage = new MonitoringPage(); - broker = settings.get( "broker" ); - brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); - namespace = settings.get( "namespace" ); + this.monitoringPage = new MonitoringPage(); + this.broker = settings.get( "broker" ); + this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); + String name = settings.get( "namespace" ); + NamespaceType type = NamespaceType.valueOf(settings.get( "namespace type" )); + if ( validateNamespaceName( name, type ) ) { + this.namespace = name; + this.namespaceType = type; + } } @@ -135,9 +145,9 @@ public void run() { } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); /**List topicsList = new ArrayList<>( List.of( this.settings.get( "topics" ).split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { - topicsList.set( i, topicsList.get(i).trim() ); - }**/ + for ( int i = 0; i < topicsList.size(); i++ ) { + topicsList.set( i, topicsList.get(i).trim() ); + }**/ subscribe( topicsToList( this.settings.get( "topics" ) ) ); } } @@ -168,10 +178,40 @@ public void shutdown() { } + + private boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { + // TODO: Nachrichten an UI schicken falls Namespace name nicht geht + boolean nameCanBeUsed = false; + Catalog catalog = Catalog.getInstance(); + // TODO: database ID evtl von UI abfragen? + if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + return nameCanBeUsed; + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + nameCanBeUsed = true; + } else { + log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); + log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); + } + } else { + nameCanBeUsed = true; + } + //TODO: rmv + log.info( String.valueOf( nameCanBeUsed ) ); + return nameCanBeUsed; + } + + public List topicsToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { - topicsList.set( i, topicsList.get(i).trim() ); + topicsList.set( i, topicsList.get( i ).trim() ); } return topicsList; } @@ -179,30 +219,40 @@ public List topicsToList( String topics ) { @Override protected void reloadSettings( List updatedSettings ) { + for ( String changedSetting : updatedSettings ) { + switch ( changedSetting ) { + case "topics": + List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); + + List topicsToSub = new ArrayList<>(); + for ( String newTopic : newTopicsList ) { + if ( !topics.contains( newTopic ) ) { + topicsToSub.add( newTopic ); + } + } - if ( updatedSettings.contains( "topics" ) ) { - List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); - - List topicsToSub = new ArrayList<>(); - for ( String newTopic : newTopicsList ) { - if ( !topics.contains( newTopic ) ) { - topicsToSub.add( newTopic ); - } - } - - if ( !topicsToSub.isEmpty() ) { - subscribe( topicsToSub ); - } + if ( !topicsToSub.isEmpty() ) { + subscribe( topicsToSub ); + } - List topicsToUnsub = new ArrayList<>(); - for ( String oldTopic : topics ) { - if ( !newTopicsList.contains( oldTopic ) ) { - topicsToUnsub.add( oldTopic ); - } - } + List topicsToUnsub = new ArrayList<>(); + for ( String oldTopic : topics ) { + if ( !newTopicsList.contains( oldTopic ) ) { + topicsToUnsub.add( oldTopic ); + } + } - if ( !topicsToUnsub.isEmpty() ) { - unsubscribe( topicsToUnsub ); + if ( !topicsToUnsub.isEmpty() ) { + unsubscribe( topicsToUnsub ); + } + break; + + case "namespace": + this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + break; + case "namespace type": + this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + break; } } @@ -261,9 +311,9 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. String content = StreamProcessing.processMsg( subMsg ); - PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace ); - StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream); - streamCapture.handleContent( ); + PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); + StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); + streamCapture.handleContent(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index fb0c7ee69e..4fed4544ab 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -19,19 +19,21 @@ import lombok.Getter; import lombok.Setter; import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; public class PolyStream { // Representation of 1 content in a namespace @Getter final String topic; - @Getter - final String uniqueNameOfInterface; + private final String uniqueNameOfInterface; + @Getter + private final String content; @Getter - final String content; + private final String namespace; @Getter - final String namespace; + private final NamespaceType namespaceType; @Getter @Setter private long namespaceID; @@ -46,57 +48,15 @@ public class PolyStream { private long storeID; // the ID of collection/graph/table... the place where info is/should be saved - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace ) { + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, NamespaceType namespaceType ) { this.topic = topic; this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; this.namespace = namespace; + this.namespaceType = namespaceType; this.databaseId = Catalog.defaultDatabaseId; this.userId = Catalog.defaultUserId; } - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId ) { - this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; - this.content = content; - this.namespace = namespace; - this.databaseId = databaseId; - this.userId = userId; - } - - - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId ) { - this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; - this.content = content; - this.namespace = namespace; - this.namespaceID = namespaceID; - this.databaseId = databaseId; - this.userId = userId; - } - - - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long databaseId, int userId, long storeID ) { - this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; - this.content = content; - this.namespace = namespace; - this.databaseId = databaseId; - this.userId = userId; - this.storeID = storeID; - } - - - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, long namespaceID, long databaseId, int userId, long storeID ) { - this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; - this.content = content; - this.namespace = namespace; - this.namespaceID = namespaceID; - this.databaseId = databaseId; - this.userId = userId; - this.storeID = storeID; - } - } From 3f8ab3cb10b513ac40eed5bc88375beff8d36828 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 9 Jul 2023 10:04:35 +0200 Subject: [PATCH 033/114] fixed minor changes Saving msg only works for DOCUMENT model schema. --- .../org/polypheny/db/mqtt/StreamCapture.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index ef3fc413b4..c5ecae71f5 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -90,11 +90,11 @@ long getCollectionID() { // check for existing namespace with DOCUMENT NamespaceType: - if ( catalog.checkIfExistsSchema( stream.getDatabaseId(), stream.getNamespace() ) ) { + if ( catalog.checkIfExistsSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ) ) { CatalogSchema schema = null; try { - schema = catalog.getSchema( stream.getDatabaseId(), stream.getNamespace() ); + schema = catalog.getSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ); } catch ( UnknownSchemaException e ) { log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); return 0; @@ -106,8 +106,8 @@ long getCollectionID() { //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic List collectionList = catalog.getCollections( schema.id, null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.uniqueNameOfInterface ).getQueryInterfaceId(); + if ( collection.name.equals( this.stream.topic ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); if ( !collection.placements.contains( queryInterfaceId ) ) { return collection.addPlacement( queryInterfaceId ).id; } else { @@ -118,20 +118,25 @@ long getCollectionID() { return createNewCollection(); } else { - this.stream.setNamespaceID( createNamespace() ); + this.stream.setNamespaceID( createNewNamespace() ); return createNewCollection(); } } else { - this.stream.setNamespaceID( createNamespace() ); + this.stream.setNamespaceID( createNewNamespace() ); return createNewCollection(); } } - private long createNamespace() { + private long createNewNamespace() { Catalog catalog = Catalog.getInstance(); - //TOdO: commit - return catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); + long namespaceId = catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); + try { + catalog.commit(); + } catch ( NoTablePrimaryKeyException e ) { + log.error( "An error " ); + } + return namespaceId; } From 805a691c42d34983532a660a6549fc8dd743eb67 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 11 Jul 2023 16:00:10 +0200 Subject: [PATCH 034/114] included MqttTopic objects changed the structure of PolyStream. Now it includes the MqttTopic Objekt and the content of a message. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 139 ++++++++++-------- .../java/org/polypheny/db/mqtt/MqttTopic.java | 94 ++++++++++++ .../org/polypheny/db/mqtt/PolyStream.java | 49 +++--- .../org/polypheny/db/mqtt/StreamCapture.java | 80 ++++++---- .../polypheny/db/mqtt/StreamProcessing.java | 17 +-- 5 files changed, 251 insertions(+), 128 deletions(-) create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 9746bf240a..6c03ea3cfd 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -61,9 +62,9 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - // TODO namespace und type müssen dringend eingegeben werden, so machen wie bei topics? - mqttDefaultSettings.put( "namespace", "public" ); - mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); + // TODO Bei run mit reset: braucht es diese default settings sonst kommt nullpointer exception. +// mqttDefaultSettings.put( "namespace", "public" ); +// mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -95,11 +96,11 @@ public static class MqttStreamServer extends QueryInterface { private final int brokerPort; - private ArrayList topics = new ArrayList(); + private List topics = new ArrayList(); private static Mqtt3AsyncClient client; - private String namespace; + private String namespaceName; private NamespaceType namespaceType; @@ -114,8 +115,8 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); String name = settings.get( "namespace" ); NamespaceType type = NamespaceType.valueOf(settings.get( "namespace type" )); - if ( validateNamespaceName( name, type ) ) { - this.namespace = name; + if ( StreamCapture.validateNamespaceName( name, type ) ) { + this.namespaceName = name; this.namespaceType = type; } } @@ -145,10 +146,10 @@ public void run() { } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); /**List topicsList = new ArrayList<>( List.of( this.settings.get( "topics" ).split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { + for ( int i = 0; i < atopicsList.size(); i++ ) { topicsList.set( i, topicsList.get(i).trim() ); }**/ - subscribe( topicsToList( this.settings.get( "topics" ) ) ); + subscribe( topicsStringToList( this.settings.get( "topics" ) ) ); } } ); @@ -179,36 +180,10 @@ public void shutdown() { } - private boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { - // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - boolean nameCanBeUsed = false; - Catalog catalog = Catalog.getInstance(); - // TODO: database ID evtl von UI abfragen? - if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { - CatalogSchema schema = null; - try { - schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); - } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - return nameCanBeUsed; - } - assert schema != null; - if ( schema.namespaceType == namespaceType ) { - nameCanBeUsed = true; - } else { - log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); - log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); - } - } else { - nameCanBeUsed = true; - } - //TODO: rmv - log.info( String.valueOf( nameCanBeUsed ) ); - return nameCanBeUsed; - } - public List topicsToList( String topics ) { + + public List topicsStringToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { topicsList.set( i, topicsList.get( i ).trim() ); @@ -222,11 +197,18 @@ protected void reloadSettings( List updatedSettings ) { for ( String changedSetting : updatedSettings ) { switch ( changedSetting ) { case "topics": - List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); + List newTopicsList = topicsStringToList( this.getCurrentSettings().get( "topics" ) ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { - if ( !topics.contains( newTopic ) ) { + boolean containedInTopics = false; + for ( MqttTopic t : topics ) { + if ( ! t.topicName.equals( newTopic ) ){ + containedInTopics = true; + break; + } + } + if ( !containedInTopics ) { topicsToSub.add( newTopic ); } } @@ -235,9 +217,16 @@ protected void reloadSettings( List updatedSettings ) { subscribe( topicsToSub ); } - List topicsToUnsub = new ArrayList<>(); - for ( String oldTopic : topics ) { - if ( !newTopicsList.contains( oldTopic ) ) { + List topicsToUnsub = new ArrayList<>(); + for ( MqttTopic oldTopic : topics ) { + boolean containedInNewTopicsList = false; + for ( String newTopic : newTopicsList ) { + if ( ! oldTopic.topicName.equals( newTopic ) ){ + containedInNewTopicsList = true; + break; + } + } + if ( !containedInNewTopicsList ) { topicsToUnsub.add( oldTopic ); } } @@ -248,10 +237,34 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + if ( StreamCapture.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), this.namespaceType ) ) { + this.namespaceName = this.getCurrentSettings().get( "namespace" ); + List changedTopics = new ArrayList<>(); + for ( MqttTopic t : topics ) { + MqttTopic newt = t.setNewNamespace( this.namespaceName, 0, this.namespaceType ); + changedTopics.add(newt); + } + this.topics = changedTopics; + log.info( "namespsace name changed" ); + } else { + //TODO: rmv + log.info( "new Name not updated in Objekts" ); + } break; case "namespace type": - this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + if ( StreamCapture.validateNamespaceName( this.namespaceName, NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ) ) { + this.namespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + List changedTopics = new ArrayList<>(); + for ( MqttTopic t : topics ) { + MqttTopic newt = t.setNewNamespace( this.namespaceName, 0, this.namespaceType ); + changedTopics.add(newt); + } + this.topics = changedTopics; + log.info( "namespsace type changed" ); + } else { + //TODO: rmv + log.info( "new Name not updated in Opjekts" ); + } break; } } @@ -267,20 +280,21 @@ void subscribe( List newTopics ) { /** - * subscribes to one given topic and adds it to the List topics. + * subscribes to given topic and adds it to the List topics. * - * @param topic the topic the client should subscribe to. + * @param topicName the topic the client should subscribe to. */ - public void subscribe( String topic ) { - client.subscribeWith().topicFilter( topic ).callback( subMsg -> { + public void subscribe( String topicName ) { + client.subscribeWith().topicFilter( topicName ).callback( subMsg -> { log.info( "Received message from topic {}.", subMsg.getTopic() ); processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { log.info( "Subscription was not successfull. Please try again." ); } else { - this.topics.add( topic ); - log.info( "Successful subscription to topic:{}.", topic ); + MqttTopic newTopic = new MqttTopic( topicName, this.namespaceName, this.namespaceType, Catalog.defaultDatabaseId, Catalog.defaultUserId, this.getQueryInterfaceId() ); + this.topics.add( newTopic ); + log.info( "Successful subscription to topic:{}.", topicName ); } } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). @@ -288,21 +302,21 @@ public void subscribe( String topic ) { } - public void unsubscribe( List topics ) { - for ( String t : topics ) { + public void unsubscribe( List topics ) { + for ( MqttTopic t : topics ) { unsubscribe( t ); } } - public void unsubscribe( String topic ) { - client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { + public void unsubscribe( MqttTopic topic ) { + client.unsubscribeWith().topicFilter( topic.topicName ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); + log.error( String.format( "Topic %s could not be unsubscribed.", topic.topicName ) ); } else { this.topics.remove( topic ); - log.info( "Unsubscribed from topic:{}.", topic ); + log.info( "Unsubscribed from topic:{}.", topic.topicName ); } } ); } @@ -310,8 +324,11 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. - String content = StreamProcessing.processMsg( subMsg ); - PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); + + //TODO: get topic from List don't create it like below + MqttTopic topic = new MqttTopic( subMsg.getTopic().toString(), this.namespaceName, this.namespaceType, Catalog.defaultDatabaseId, Catalog.defaultUserId, this.getQueryInterfaceId() ); + String content = StreamProcessing.processMsg( subMsg.getPayloadAsBytes().toString(),topic ); + PolyStream stream = new PolyStream( topic, content ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); streamCapture.handleContent(); } @@ -350,7 +367,7 @@ public MonitoringPage() { // table to display topics topicsTable = new InformationTable( informationGroupTopics, - List.of( "Topics" ) + List.of( "Topics", "Message Count" ) ); im.registerInformation( topicsTable ); @@ -360,13 +377,17 @@ public MonitoringPage() { public void update() { + //TODO: Message Count topicsTable.reset(); if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { + //TODO: korrect + /** for ( String topic : topics ) { topicsTable.addRow( topic ); } + **/ } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java new file mode 100644 index 0000000000..01989fb741 --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import java.util.ArrayList; +import java.util.List; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.entity.CatalogObject; + +public class MqttTopic { + + // TODO: Überlegen, wie man das alles persistent machen kann + final String topicName; + final String namespaceName; + final long namespaceId; + final NamespaceType namespaceType; + final long databaseId; + final int userId; + final int queryInterfaceId; + long storeID = 0; // the ID of collection/graph/table... the place where info is/should be saved + CatalogObject storage = null; + int msgCount; //TODO: make persistent + + + public MqttTopic( String topicName, String namespaceName, NamespaceType namespaceType, long databaseId, int userId, int queryInterfaceId ) { + this.topicName = topicName; + this.namespaceName = namespaceName; + this.namespaceId = 0; + this.namespaceType = namespaceType; + this.databaseId = databaseId; + this.userId = userId; + this.queryInterfaceId = queryInterfaceId; + } + + + public MqttTopic( String topicName, String namespaceName, long namespaceId, NamespaceType namespaceType, long databaseId, int userId, int queryInterfaceId ) { + this.topicName = topicName; + this.namespaceName = namespaceName; + this.namespaceId = namespaceId; + this.namespaceType = namespaceType; + this.databaseId = databaseId; + this.userId = userId; + this.queryInterfaceId = queryInterfaceId; + } + + + public MqttTopic setDatabaseId( long databaseId ) { + return new MqttTopic( this.topicName, this.namespaceName, this.namespaceId, this.namespaceType, databaseId, this.userId, this.queryInterfaceId ); + } + + + public MqttTopic setUserId( int userId ) { + return new MqttTopic( this.topicName, this.namespaceName, this.namespaceId, this.namespaceType, this.databaseId, userId, this.queryInterfaceId ); + } + + + /** + * @param newId 0 if namespaceId is not known yet. + */ + public MqttTopic setNewNamespace( String newName, long newId, NamespaceType type ) { + return new MqttTopic( this.topicName, newName, newId, type, this.databaseId, this.userId, this.queryInterfaceId ); + } + + + public MqttTopic setNamespaceId( long id ) { + return new MqttTopic( this.topicName, this.namespaceName, id, this.namespaceType, this.databaseId, this.userId, this.queryInterfaceId ); + } + + + public void increaseMsgCount() { + ++this.msgCount; + } + public List getRecentMessages() { + //TODO: implement + List msgList = new ArrayList<>(); + msgList.add( "msg1" ); + msgList.add( "msg2" ); + return msgList; + } +} diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index 4fed4544ab..6b43462c38 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -18,44 +18,41 @@ import lombok.Getter; import lombok.Setter; -import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; public class PolyStream { // Representation of 1 content in a namespace @Getter - final String topic; - @Getter - private final String uniqueNameOfInterface; - @Getter - private final String content; - @Getter - private final String namespace; - @Getter - private final NamespaceType namespaceType; - @Getter - @Setter - private long namespaceID; - @Setter - @Getter - private long databaseId; @Setter + private MqttTopic topic; @Getter - private int userId; - @Getter - @Setter - private long storeID; // the ID of collection/graph/table... the place where info is/should be saved + private final String content; - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, NamespaceType namespaceType ) { + public PolyStream( MqttTopic topic, String content ) { this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; - this.namespace = namespace; - this.namespaceType = namespaceType; - this.databaseId = Catalog.defaultDatabaseId; - this.userId = Catalog.defaultUserId; + + } + + public void setDatabaseId( long databaseId ) { + this.topic = this.topic.setDatabaseId( databaseId ); + } + + + public void setUserId( int userId ) { + this.topic = this.topic.setUserId( userId ); + } + + + public void setNewNameSpace( String newName, long newId, NamespaceType type ) { + this.topic = this.topic.setNewNamespace( newName, newId, type ); + } + + + public void setNamespaceId( long namespaceId ) { + this.topic = this.topic.setNamespaceId( namespaceId ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index c5ecae71f5..a49e8f793b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -66,12 +66,40 @@ public class StreamCapture { this.stream.setDatabaseId( statement.getPrepareContext().getDatabaseId() ); } + public static boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { + // TODO: Nachrichten an UI schicken falls Namespace name nicht geht + boolean nameCanBeUsed = false; + Catalog catalog = Catalog.getInstance(); + // TODO: database ID evtl von von statement.getPrepareContext().getCurrentUserId() und statement.getPrepareContext().getDatabaseId() nehmen? + if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + return nameCanBeUsed; + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + nameCanBeUsed = true; + } else { + log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); + log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); + } + } else { + nameCanBeUsed = true; + } + //TODO: rmv + log.info( String.valueOf( nameCanBeUsed ) ); + return nameCanBeUsed; + } + - public void handleContent() { + void handleContent() { //String path = registerTopicFolder(topic); long storeId = getCollectionID(); if ( storeId != 0 ) { - stream.setStoreID( storeId ); + stream.getTopic().storeID = storeId; boolean saved = saveContent(); //TODO: gescheite Tests // Catalog catalog = Catalog.getInstance(); @@ -86,15 +114,17 @@ public void handleContent() { */ long getCollectionID() { Catalog catalog = Catalog.getInstance(); - List schemaList = catalog.getSchemas( this.stream.getDatabaseId(), null ); + List schemaList = catalog.getSchemas( this.stream.getTopic().databaseId, null ); // check for existing namespace with DOCUMENT NamespaceType: - if ( catalog.checkIfExistsSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ) ) { + //TODO: if wird nur gemacht, wenn Topic das erste mal Nachricht bekommt und noch kein storage zugewiesen bekommen hat. + // DH: wenn CatalogObject oder storeID noch null ist, dann diese ganze Abfrage machen. + if ( catalog.checkIfExistsSchema( this.stream.getTopic().databaseId, this.stream.getTopic().namespaceName ) ) { CatalogSchema schema = null; try { - schema = catalog.getSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ); + schema = catalog.getSchema( this.stream.getTopic().databaseId, this.stream.getTopic().namespaceName ); } catch ( UnknownSchemaException e ) { log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); return 0; @@ -102,14 +132,13 @@ long getCollectionID() { assert schema != null; if ( schema.namespaceType == NamespaceType.DOCUMENT ) { - this.stream.setNamespaceID( schema.id ); + this.stream.setTopic( this.stream.getTopic().setNamespaceId( schema.id ) ); //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic List collectionList = catalog.getCollections( schema.id, null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); - if ( !collection.placements.contains( queryInterfaceId ) ) { - return collection.addPlacement( queryInterfaceId ).id; + if ( collection.name.equals( this.stream.getTopic().topicName ) ) { + if ( !collection.placements.contains( this.stream.getTopic().queryInterfaceId ) ) { + return collection.addPlacement( this.stream.getTopic().queryInterfaceId ).id; } else { return collection.id; } @@ -118,19 +147,19 @@ long getCollectionID() { return createNewCollection(); } else { - this.stream.setNamespaceID( createNewNamespace() ); + this.stream.setNamespaceId( addNewNamespace() ); return createNewCollection(); } } else { - this.stream.setNamespaceID( createNewNamespace() ); + this.stream.setNamespaceId( addNewNamespace() ); return createNewCollection(); } } - private long createNewNamespace() { + private long addNewNamespace() { Catalog catalog = Catalog.getInstance(); - long namespaceId = catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); + long namespaceId = catalog.addNamespace( this.stream.getTopic().namespaceName, this.stream.getTopic().databaseId, this.stream.getTopic().userId, NamespaceType.DOCUMENT ); try { catalog.commit(); } catch ( NoTablePrimaryKeyException e ) { @@ -150,13 +179,13 @@ private long createNewCollection() { try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( - this.stream.getNamespaceID(), - this.stream.topic, + this.stream.getTopic().namespaceId, + this.stream.getTopic().topicName, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created Collection with name: {}", this.stream.topic ); + log.info( "Created Collection with name: {}", this.stream.getTopic().topicName ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collaction already existing with this name." ); @@ -166,11 +195,10 @@ private long createNewCollection() { return 0; } //add placement - List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); + List collectionList = catalog.getCollections( this.stream.getTopic().namespaceId, null ); for ( int i = 0; i < collectionList.size(); i++ ) { - if ( collectionList.get( i ).name.equals( this.stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); - collectionList.set( i, collectionList.get( i ).addPlacement( queryInterfaceId ) ); + if ( collectionList.get( i ).name.equals( this.stream.getTopic().topicName ) ) { + collectionList.set( i, collectionList.get( i ).addPlacement( this.stream.getTopic().queryInterfaceId ) ); return collectionList.get( i ).id; } @@ -282,7 +310,7 @@ boolean saveContent() { private Transaction getTransaction() { try { - return transactionManager.startTransaction( this.stream.getUserId(), this.stream.getDatabaseId(), false, "MQTT Stream" ); + return transactionManager.startTransaction( this.stream.getTopic().userId, this.stream.getTopic().databaseId, false, "MQTT Stream" ); } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { throw new RuntimeException( "Error while starting transaction", e ); } @@ -337,15 +365,7 @@ private static String registerTopicFolder( String topic ) { //TODO: rmv log log.info( "Directory already exists" ); } - return path; - } - - public static void main( String[] args ) { - - } - - } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java index 2d61e3effe..462324da79 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -16,19 +16,16 @@ package org.polypheny.db.mqtt; -import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; -import java.nio.charset.Charset; import lombok.extern.slf4j.Slf4j; @Slf4j public class StreamProcessing { - public static String processMsg( Mqtt3Publish subMsg ) { - String msg = toString( subMsg ); - - String info = extractInfo( msg ); + public static String processMsg( String subMsg, MqttTopic topic ) { + String info = extractInfo( subMsg ); if ( validateMsg( info ) ) { - log.info( "Extracted and validated message: {}", msg); + log.info( "Extracted and validated message: {}", subMsg); + topic.increaseMsgCount(); return info; } else { log.error( "Message is not valid!" ); @@ -36,12 +33,6 @@ public static String processMsg( Mqtt3Publish subMsg ) { } } - - private static String toString( Mqtt3Publish subMsg ) { - return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); - } - - private static boolean validateMsg( String msg ) { //TODO: Implement return true; From 9fe13b402f64bbdafb31d5e87848ffb961aacff3 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 11 Jul 2023 19:15:38 +0200 Subject: [PATCH 035/114] debugging of reloadSettings method --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 6c03ea3cfd..2b7b8b6ef0 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -21,6 +21,7 @@ import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -37,6 +38,8 @@ import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.information.InformationAction; +import org.polypheny.db.information.InformationAction.Action; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; @@ -203,7 +206,7 @@ protected void reloadSettings( List updatedSettings ) { for ( String newTopic : newTopicsList ) { boolean containedInTopics = false; for ( MqttTopic t : topics ) { - if ( ! t.topicName.equals( newTopic ) ){ + if ( t.topicName.equals( newTopic ) ){ containedInTopics = true; break; } @@ -221,7 +224,7 @@ protected void reloadSettings( List updatedSettings ) { for ( MqttTopic oldTopic : topics ) { boolean containedInNewTopicsList = false; for ( String newTopic : newTopicsList ) { - if ( ! oldTopic.topicName.equals( newTopic ) ){ + if ( oldTopic.topicName.equals( newTopic ) ){ containedInNewTopicsList = true; break; } @@ -245,10 +248,6 @@ protected void reloadSettings( List updatedSettings ) { changedTopics.add(newt); } this.topics = changedTopics; - log.info( "namespsace name changed" ); - } else { - //TODO: rmv - log.info( "new Name not updated in Objekts" ); } break; case "namespace type": @@ -260,10 +259,6 @@ protected void reloadSettings( List updatedSettings ) { changedTopics.add(newt); } this.topics = changedTopics; - log.info( "namespsace type changed" ); - } else { - //TODO: rmv - log.info( "new Name not updated in Opjekts" ); } break; } @@ -352,6 +347,10 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; + private final InformationGroup informationGroupMsg; + + private final InformationAction msgButton; + private final InformationTable topicsTable; @@ -373,6 +372,17 @@ public MonitoringPage() { im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); + //TODO: rmv button + informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 2 ); + im.addGroup( informationGroupMsg ); + + msgButton = new InformationAction( informationGroupMsg, "Send a msg", (parameters) -> { + String end = "Msg was published!"; + client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes( StandardCharsets.UTF_8 ) ).send(); + return end; + }).withParameters( "topic", "msg" ); + im.registerInformation( msgButton ); + } From e4e9d0b33d2190c81cbaeeb84ad40bab113bde7e Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Wed, 12 Jul 2023 16:40:50 +0200 Subject: [PATCH 036/114] Revert "debugging of reloadSettings method" This reverts commit 9fe13b402f64bbdafb31d5e87848ffb961aacff3. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 2b7b8b6ef0..6c03ea3cfd 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -21,7 +21,6 @@ import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -38,8 +37,6 @@ import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; -import org.polypheny.db.information.InformationAction; -import org.polypheny.db.information.InformationAction.Action; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; @@ -206,7 +203,7 @@ protected void reloadSettings( List updatedSettings ) { for ( String newTopic : newTopicsList ) { boolean containedInTopics = false; for ( MqttTopic t : topics ) { - if ( t.topicName.equals( newTopic ) ){ + if ( ! t.topicName.equals( newTopic ) ){ containedInTopics = true; break; } @@ -224,7 +221,7 @@ protected void reloadSettings( List updatedSettings ) { for ( MqttTopic oldTopic : topics ) { boolean containedInNewTopicsList = false; for ( String newTopic : newTopicsList ) { - if ( oldTopic.topicName.equals( newTopic ) ){ + if ( ! oldTopic.topicName.equals( newTopic ) ){ containedInNewTopicsList = true; break; } @@ -248,6 +245,10 @@ protected void reloadSettings( List updatedSettings ) { changedTopics.add(newt); } this.topics = changedTopics; + log.info( "namespsace name changed" ); + } else { + //TODO: rmv + log.info( "new Name not updated in Objekts" ); } break; case "namespace type": @@ -259,6 +260,10 @@ protected void reloadSettings( List updatedSettings ) { changedTopics.add(newt); } this.topics = changedTopics; + log.info( "namespsace type changed" ); + } else { + //TODO: rmv + log.info( "new Name not updated in Opjekts" ); } break; } @@ -347,10 +352,6 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; - private final InformationGroup informationGroupMsg; - - private final InformationAction msgButton; - private final InformationTable topicsTable; @@ -372,17 +373,6 @@ public MonitoringPage() { im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); - //TODO: rmv button - informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 2 ); - im.addGroup( informationGroupMsg ); - - msgButton = new InformationAction( informationGroupMsg, "Send a msg", (parameters) -> { - String end = "Msg was published!"; - client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes( StandardCharsets.UTF_8 ) ).send(); - return end; - }).withParameters( "topic", "msg" ); - im.registerInformation( msgButton ); - } From 68be5dd7a9ab27dad9dc882ae49007d881a63004 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Wed, 12 Jul 2023 16:57:21 +0200 Subject: [PATCH 037/114] Revert "included MqttTopic objects" This reverts commit 805a691c42d34983532a660a6549fc8dd743eb67. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 139 ++++++++---------- .../java/org/polypheny/db/mqtt/MqttTopic.java | 94 ------------ .../org/polypheny/db/mqtt/PolyStream.java | 49 +++--- .../org/polypheny/db/mqtt/StreamCapture.java | 80 ++++------ .../polypheny/db/mqtt/StreamProcessing.java | 17 ++- 5 files changed, 128 insertions(+), 251 deletions(-) delete mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 6c03ea3cfd..9746bf240a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -25,7 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -62,9 +61,9 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - // TODO Bei run mit reset: braucht es diese default settings sonst kommt nullpointer exception. -// mqttDefaultSettings.put( "namespace", "public" ); -// mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); + // TODO namespace und type müssen dringend eingegeben werden, so machen wie bei topics? + mqttDefaultSettings.put( "namespace", "public" ); + mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -96,11 +95,11 @@ public static class MqttStreamServer extends QueryInterface { private final int brokerPort; - private List topics = new ArrayList(); + private ArrayList topics = new ArrayList(); private static Mqtt3AsyncClient client; - private String namespaceName; + private String namespace; private NamespaceType namespaceType; @@ -115,8 +114,8 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); String name = settings.get( "namespace" ); NamespaceType type = NamespaceType.valueOf(settings.get( "namespace type" )); - if ( StreamCapture.validateNamespaceName( name, type ) ) { - this.namespaceName = name; + if ( validateNamespaceName( name, type ) ) { + this.namespace = name; this.namespaceType = type; } } @@ -146,10 +145,10 @@ public void run() { } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); /**List topicsList = new ArrayList<>( List.of( this.settings.get( "topics" ).split( "," ) ) ); - for ( int i = 0; i < atopicsList.size(); i++ ) { + for ( int i = 0; i < topicsList.size(); i++ ) { topicsList.set( i, topicsList.get(i).trim() ); }**/ - subscribe( topicsStringToList( this.settings.get( "topics" ) ) ); + subscribe( topicsToList( this.settings.get( "topics" ) ) ); } } ); @@ -180,10 +179,36 @@ public void shutdown() { } + private boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { + // TODO: Nachrichten an UI schicken falls Namespace name nicht geht + boolean nameCanBeUsed = false; + Catalog catalog = Catalog.getInstance(); + // TODO: database ID evtl von UI abfragen? + if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); + return nameCanBeUsed; + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + nameCanBeUsed = true; + } else { + log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); + log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); + } + } else { + nameCanBeUsed = true; + } + //TODO: rmv + log.info( String.valueOf( nameCanBeUsed ) ); + return nameCanBeUsed; + } - - public List topicsStringToList( String topics ) { + public List topicsToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { topicsList.set( i, topicsList.get( i ).trim() ); @@ -197,18 +222,11 @@ protected void reloadSettings( List updatedSettings ) { for ( String changedSetting : updatedSettings ) { switch ( changedSetting ) { case "topics": - List newTopicsList = topicsStringToList( this.getCurrentSettings().get( "topics" ) ); + List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { - boolean containedInTopics = false; - for ( MqttTopic t : topics ) { - if ( ! t.topicName.equals( newTopic ) ){ - containedInTopics = true; - break; - } - } - if ( !containedInTopics ) { + if ( !topics.contains( newTopic ) ) { topicsToSub.add( newTopic ); } } @@ -217,16 +235,9 @@ protected void reloadSettings( List updatedSettings ) { subscribe( topicsToSub ); } - List topicsToUnsub = new ArrayList<>(); - for ( MqttTopic oldTopic : topics ) { - boolean containedInNewTopicsList = false; - for ( String newTopic : newTopicsList ) { - if ( ! oldTopic.topicName.equals( newTopic ) ){ - containedInNewTopicsList = true; - break; - } - } - if ( !containedInNewTopicsList ) { + List topicsToUnsub = new ArrayList<>(); + for ( String oldTopic : topics ) { + if ( !newTopicsList.contains( oldTopic ) ) { topicsToUnsub.add( oldTopic ); } } @@ -237,34 +248,10 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - if ( StreamCapture.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), this.namespaceType ) ) { - this.namespaceName = this.getCurrentSettings().get( "namespace" ); - List changedTopics = new ArrayList<>(); - for ( MqttTopic t : topics ) { - MqttTopic newt = t.setNewNamespace( this.namespaceName, 0, this.namespaceType ); - changedTopics.add(newt); - } - this.topics = changedTopics; - log.info( "namespsace name changed" ); - } else { - //TODO: rmv - log.info( "new Name not updated in Objekts" ); - } + this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); break; case "namespace type": - if ( StreamCapture.validateNamespaceName( this.namespaceName, NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ) ) { - this.namespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - List changedTopics = new ArrayList<>(); - for ( MqttTopic t : topics ) { - MqttTopic newt = t.setNewNamespace( this.namespaceName, 0, this.namespaceType ); - changedTopics.add(newt); - } - this.topics = changedTopics; - log.info( "namespsace type changed" ); - } else { - //TODO: rmv - log.info( "new Name not updated in Opjekts" ); - } + this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); break; } } @@ -280,21 +267,20 @@ void subscribe( List newTopics ) { /** - * subscribes to given topic and adds it to the List topics. + * subscribes to one given topic and adds it to the List topics. * - * @param topicName the topic the client should subscribe to. + * @param topic the topic the client should subscribe to. */ - public void subscribe( String topicName ) { - client.subscribeWith().topicFilter( topicName ).callback( subMsg -> { + public void subscribe( String topic ) { + client.subscribeWith().topicFilter( topic ).callback( subMsg -> { log.info( "Received message from topic {}.", subMsg.getTopic() ); processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { log.info( "Subscription was not successfull. Please try again." ); } else { - MqttTopic newTopic = new MqttTopic( topicName, this.namespaceName, this.namespaceType, Catalog.defaultDatabaseId, Catalog.defaultUserId, this.getQueryInterfaceId() ); - this.topics.add( newTopic ); - log.info( "Successful subscription to topic:{}.", topicName ); + this.topics.add( topic ); + log.info( "Successful subscription to topic:{}.", topic ); } } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). @@ -302,21 +288,21 @@ public void subscribe( String topicName ) { } - public void unsubscribe( List topics ) { - for ( MqttTopic t : topics ) { + public void unsubscribe( List topics ) { + for ( String t : topics ) { unsubscribe( t ); } } - public void unsubscribe( MqttTopic topic ) { - client.unsubscribeWith().topicFilter( topic.topicName ).send().whenComplete( ( unsub, throwable ) -> { + public void unsubscribe( String topic ) { + client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - log.error( String.format( "Topic %s could not be unsubscribed.", topic.topicName ) ); + log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { this.topics.remove( topic ); - log.info( "Unsubscribed from topic:{}.", topic.topicName ); + log.info( "Unsubscribed from topic:{}.", topic ); } } ); } @@ -324,11 +310,8 @@ public void unsubscribe( MqttTopic topic ) { void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. - - //TODO: get topic from List don't create it like below - MqttTopic topic = new MqttTopic( subMsg.getTopic().toString(), this.namespaceName, this.namespaceType, Catalog.defaultDatabaseId, Catalog.defaultUserId, this.getQueryInterfaceId() ); - String content = StreamProcessing.processMsg( subMsg.getPayloadAsBytes().toString(),topic ); - PolyStream stream = new PolyStream( topic, content ); + String content = StreamProcessing.processMsg( subMsg ); + PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); streamCapture.handleContent(); } @@ -367,7 +350,7 @@ public MonitoringPage() { // table to display topics topicsTable = new InformationTable( informationGroupTopics, - List.of( "Topics", "Message Count" ) + List.of( "Topics" ) ); im.registerInformation( topicsTable ); @@ -377,17 +360,13 @@ public MonitoringPage() { public void update() { - //TODO: Message Count topicsTable.reset(); if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - //TODO: korrect - /** for ( String topic : topics ) { topicsTable.addRow( topic ); } - **/ } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java deleted file mode 100644 index 01989fb741..0000000000 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttTopic.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.mqtt; - -import java.util.ArrayList; -import java.util.List; -import org.polypheny.db.catalog.Catalog.NamespaceType; -import org.polypheny.db.catalog.entity.CatalogObject; - -public class MqttTopic { - - // TODO: Überlegen, wie man das alles persistent machen kann - final String topicName; - final String namespaceName; - final long namespaceId; - final NamespaceType namespaceType; - final long databaseId; - final int userId; - final int queryInterfaceId; - long storeID = 0; // the ID of collection/graph/table... the place where info is/should be saved - CatalogObject storage = null; - int msgCount; //TODO: make persistent - - - public MqttTopic( String topicName, String namespaceName, NamespaceType namespaceType, long databaseId, int userId, int queryInterfaceId ) { - this.topicName = topicName; - this.namespaceName = namespaceName; - this.namespaceId = 0; - this.namespaceType = namespaceType; - this.databaseId = databaseId; - this.userId = userId; - this.queryInterfaceId = queryInterfaceId; - } - - - public MqttTopic( String topicName, String namespaceName, long namespaceId, NamespaceType namespaceType, long databaseId, int userId, int queryInterfaceId ) { - this.topicName = topicName; - this.namespaceName = namespaceName; - this.namespaceId = namespaceId; - this.namespaceType = namespaceType; - this.databaseId = databaseId; - this.userId = userId; - this.queryInterfaceId = queryInterfaceId; - } - - - public MqttTopic setDatabaseId( long databaseId ) { - return new MqttTopic( this.topicName, this.namespaceName, this.namespaceId, this.namespaceType, databaseId, this.userId, this.queryInterfaceId ); - } - - - public MqttTopic setUserId( int userId ) { - return new MqttTopic( this.topicName, this.namespaceName, this.namespaceId, this.namespaceType, this.databaseId, userId, this.queryInterfaceId ); - } - - - /** - * @param newId 0 if namespaceId is not known yet. - */ - public MqttTopic setNewNamespace( String newName, long newId, NamespaceType type ) { - return new MqttTopic( this.topicName, newName, newId, type, this.databaseId, this.userId, this.queryInterfaceId ); - } - - - public MqttTopic setNamespaceId( long id ) { - return new MqttTopic( this.topicName, this.namespaceName, id, this.namespaceType, this.databaseId, this.userId, this.queryInterfaceId ); - } - - - public void increaseMsgCount() { - ++this.msgCount; - } - public List getRecentMessages() { - //TODO: implement - List msgList = new ArrayList<>(); - msgList.add( "msg1" ); - msgList.add( "msg2" ); - return msgList; - } -} diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java index 6b43462c38..4fed4544ab 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java @@ -18,41 +18,44 @@ import lombok.Getter; import lombok.Setter; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; public class PolyStream { // Representation of 1 content in a namespace @Getter - @Setter - private MqttTopic topic; + final String topic; + @Getter + private final String uniqueNameOfInterface; @Getter private final String content; + @Getter + private final String namespace; + @Getter + private final NamespaceType namespaceType; + @Getter + @Setter + private long namespaceID; + @Setter + @Getter + private long databaseId; + @Setter + @Getter + private int userId; + @Getter + @Setter + private long storeID; // the ID of collection/graph/table... the place where info is/should be saved - public PolyStream( MqttTopic topic, String content ) { + public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, NamespaceType namespaceType ) { this.topic = topic; + this.uniqueNameOfInterface = uniqueNameInterface; this.content = content; - - } - - public void setDatabaseId( long databaseId ) { - this.topic = this.topic.setDatabaseId( databaseId ); - } - - - public void setUserId( int userId ) { - this.topic = this.topic.setUserId( userId ); - } - - - public void setNewNameSpace( String newName, long newId, NamespaceType type ) { - this.topic = this.topic.setNewNamespace( newName, newId, type ); - } - - - public void setNamespaceId( long namespaceId ) { - this.topic = this.topic.setNamespaceId( namespaceId ); + this.namespace = namespace; + this.namespaceType = namespaceType; + this.databaseId = Catalog.defaultDatabaseId; + this.userId = Catalog.defaultUserId; } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index a49e8f793b..c5ecae71f5 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -66,40 +66,12 @@ public class StreamCapture { this.stream.setDatabaseId( statement.getPrepareContext().getDatabaseId() ); } - public static boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { - // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - boolean nameCanBeUsed = false; - Catalog catalog = Catalog.getInstance(); - // TODO: database ID evtl von von statement.getPrepareContext().getCurrentUserId() und statement.getPrepareContext().getDatabaseId() nehmen? - if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { - CatalogSchema schema = null; - try { - schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); - } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - return nameCanBeUsed; - } - assert schema != null; - if ( schema.namespaceType == namespaceType ) { - nameCanBeUsed = true; - } else { - log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); - log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); - } - } else { - nameCanBeUsed = true; - } - //TODO: rmv - log.info( String.valueOf( nameCanBeUsed ) ); - return nameCanBeUsed; - } - - void handleContent() { + public void handleContent() { //String path = registerTopicFolder(topic); long storeId = getCollectionID(); if ( storeId != 0 ) { - stream.getTopic().storeID = storeId; + stream.setStoreID( storeId ); boolean saved = saveContent(); //TODO: gescheite Tests // Catalog catalog = Catalog.getInstance(); @@ -114,17 +86,15 @@ void handleContent() { */ long getCollectionID() { Catalog catalog = Catalog.getInstance(); - List schemaList = catalog.getSchemas( this.stream.getTopic().databaseId, null ); + List schemaList = catalog.getSchemas( this.stream.getDatabaseId(), null ); // check for existing namespace with DOCUMENT NamespaceType: - //TODO: if wird nur gemacht, wenn Topic das erste mal Nachricht bekommt und noch kein storage zugewiesen bekommen hat. - // DH: wenn CatalogObject oder storeID noch null ist, dann diese ganze Abfrage machen. - if ( catalog.checkIfExistsSchema( this.stream.getTopic().databaseId, this.stream.getTopic().namespaceName ) ) { + if ( catalog.checkIfExistsSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ) ) { CatalogSchema schema = null; try { - schema = catalog.getSchema( this.stream.getTopic().databaseId, this.stream.getTopic().namespaceName ); + schema = catalog.getSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ); } catch ( UnknownSchemaException e ) { log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); return 0; @@ -132,13 +102,14 @@ long getCollectionID() { assert schema != null; if ( schema.namespaceType == NamespaceType.DOCUMENT ) { - this.stream.setTopic( this.stream.getTopic().setNamespaceId( schema.id ) ); + this.stream.setNamespaceID( schema.id ); //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic List collectionList = catalog.getCollections( schema.id, null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.stream.getTopic().topicName ) ) { - if ( !collection.placements.contains( this.stream.getTopic().queryInterfaceId ) ) { - return collection.addPlacement( this.stream.getTopic().queryInterfaceId ).id; + if ( collection.name.equals( this.stream.topic ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + if ( !collection.placements.contains( queryInterfaceId ) ) { + return collection.addPlacement( queryInterfaceId ).id; } else { return collection.id; } @@ -147,19 +118,19 @@ long getCollectionID() { return createNewCollection(); } else { - this.stream.setNamespaceId( addNewNamespace() ); + this.stream.setNamespaceID( createNewNamespace() ); return createNewCollection(); } } else { - this.stream.setNamespaceId( addNewNamespace() ); + this.stream.setNamespaceID( createNewNamespace() ); return createNewCollection(); } } - private long addNewNamespace() { + private long createNewNamespace() { Catalog catalog = Catalog.getInstance(); - long namespaceId = catalog.addNamespace( this.stream.getTopic().namespaceName, this.stream.getTopic().databaseId, this.stream.getTopic().userId, NamespaceType.DOCUMENT ); + long namespaceId = catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); try { catalog.commit(); } catch ( NoTablePrimaryKeyException e ) { @@ -179,13 +150,13 @@ private long createNewCollection() { try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( - this.stream.getTopic().namespaceId, - this.stream.getTopic().topicName, + this.stream.getNamespaceID(), + this.stream.topic, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created Collection with name: {}", this.stream.getTopic().topicName ); + log.info( "Created Collection with name: {}", this.stream.topic ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collaction already existing with this name." ); @@ -195,10 +166,11 @@ private long createNewCollection() { return 0; } //add placement - List collectionList = catalog.getCollections( this.stream.getTopic().namespaceId, null ); + List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); for ( int i = 0; i < collectionList.size(); i++ ) { - if ( collectionList.get( i ).name.equals( this.stream.getTopic().topicName ) ) { - collectionList.set( i, collectionList.get( i ).addPlacement( this.stream.getTopic().queryInterfaceId ) ); + if ( collectionList.get( i ).name.equals( this.stream.topic ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + collectionList.set( i, collectionList.get( i ).addPlacement( queryInterfaceId ) ); return collectionList.get( i ).id; } @@ -310,7 +282,7 @@ boolean saveContent() { private Transaction getTransaction() { try { - return transactionManager.startTransaction( this.stream.getTopic().userId, this.stream.getTopic().databaseId, false, "MQTT Stream" ); + return transactionManager.startTransaction( this.stream.getUserId(), this.stream.getDatabaseId(), false, "MQTT Stream" ); } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { throw new RuntimeException( "Error while starting transaction", e ); } @@ -365,7 +337,15 @@ private static String registerTopicFolder( String topic ) { //TODO: rmv log log.info( "Directory already exists" ); } + return path; + } + + public static void main( String[] args ) { + + } + + } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java index 462324da79..2d61e3effe 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -16,16 +16,19 @@ package org.polypheny.db.mqtt; +import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import java.nio.charset.Charset; import lombok.extern.slf4j.Slf4j; @Slf4j public class StreamProcessing { - public static String processMsg( String subMsg, MqttTopic topic ) { - String info = extractInfo( subMsg ); + public static String processMsg( Mqtt3Publish subMsg ) { + String msg = toString( subMsg ); + + String info = extractInfo( msg ); if ( validateMsg( info ) ) { - log.info( "Extracted and validated message: {}", subMsg); - topic.increaseMsgCount(); + log.info( "Extracted and validated message: {}", msg); return info; } else { log.error( "Message is not valid!" ); @@ -33,6 +36,12 @@ public static String processMsg( String subMsg, MqttTopic topic ) { } } + + private static String toString( Mqtt3Publish subMsg ) { + return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); + } + + private static boolean validateMsg( String msg ) { //TODO: Implement return true; From adf1278050043df5e32a2d30def7944f8ac5f7a6 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 13 Jul 2023 16:13:41 +0200 Subject: [PATCH 038/114] reverted changes made because of MqttTopic --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 111 ++++++++--- .../org/polypheny/db/mqtt/StreamCapture.java | 184 ++++-------------- .../polypheny/db/mqtt/StreamProcessing.java | 2 +- 3 files changed, 122 insertions(+), 175 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 9746bf240a..1c19e3347f 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -32,14 +32,21 @@ import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.GenericCatalogException; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; +import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; +import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.information.InformationAction; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; @@ -101,8 +108,14 @@ public static class MqttStreamServer extends QueryInterface { private String namespace; + private long namespaceId; + private NamespaceType namespaceType; + private long databaseId; + + private int userId; + private final MonitoringPage monitoringPage; @@ -112,12 +125,24 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.monitoringPage = new MonitoringPage(); this.broker = settings.get( "broker" ); this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); + //TODO: currently: setting user and database with default value from Catalog for getTransaction method -> E schönerer Weg? + // then current databaseid and userid are set. + this.databaseId = Catalog.defaultDatabaseId; + this.userId = Catalog.defaultUserId; + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + this.userId = statement.getPrepareContext().getCurrentUserId(); + this.databaseId = statement.getPrepareContext().getDatabaseId(); + String name = settings.get( "namespace" ); NamespaceType type = NamespaceType.valueOf(settings.get( "namespace type" )); - if ( validateNamespaceName( name, type ) ) { + long namespaceId = getNamespaceId( name, type ); + if ( namespaceId != 0 ) { this.namespace = name; this.namespaceType = type; + this.namespaceId = namespaceId; } + } @@ -179,41 +204,36 @@ public void shutdown() { } - private boolean validateNamespaceName( String namespaceName, NamespaceType namespaceType ) { + private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - boolean nameCanBeUsed = false; + long namespaceId = 0; + Catalog catalog = Catalog.getInstance(); - // TODO: database ID evtl von UI abfragen? - if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + if ( catalog.checkIfExistsSchema( this.databaseId, namespaceName ) ) { CatalogSchema schema = null; try { - schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + schema = catalog.getSchema( this.databaseId, namespaceName ); } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - return nameCanBeUsed; + throw new RuntimeException(e); } assert schema != null; if ( schema.namespaceType == namespaceType ) { - nameCanBeUsed = true; + namespaceId = schema.id; } else { log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); } } else { - nameCanBeUsed = true; - } - //TODO: rmv - log.info( String.valueOf( nameCanBeUsed ) ); - return nameCanBeUsed; - } - - public List topicsToList( String topics ) { - List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { - topicsList.set( i, topicsList.get( i ).trim() ); + long id = catalog.addNamespace( namespaceName, this.databaseId, this.userId, namespaceType ); + try { + catalog.commit(); + namespaceId = id; + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException(e); + } } - return topicsList; + return namespaceId; } @@ -248,14 +268,24 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + String newNamespaceName = this.getCurrentSettings().get( "namespace" ); + long namespaceId1 = this.getNamespaceId( newNamespaceName, this.namespaceType); + if ( namespaceId1 != 0 ) { + this.namespaceId = namespaceId1; + this.namespace = newNamespaceName; + } break; case "namespace type": - this.validateNamespaceName( this.getCurrentSettings().get( "namespace" ), NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ) ); + NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + long namespaceId2 = this.getNamespaceId( this.namespace, newNamespaceType); + if ( namespaceId2 != 0 ) { + this.namespaceId = namespaceId2; + this.namespaceType = newNamespaceType; + } break; + //TODO: handle change of Database maybe? } } - } @@ -312,11 +342,30 @@ void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. String content = StreamProcessing.processMsg( subMsg ); PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); + stream.setNamespaceID( this.namespaceId ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); streamCapture.handleContent(); } + public List topicsToList( String topics ) { + List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); + for ( int i = 0; i < topicsList.size(); i++ ) { + topicsList.set( i, topicsList.get( i ).trim() ); + } + return topicsList; + } + + + private Transaction getTransaction() { + try { + return transactionManager.startTransaction( this.userId, this.databaseId, false, "MQTT Stream" ); + } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { + throw new RuntimeException( "Error while starting transaction", e ); + } + } + + @Override public void languageChange() { @@ -335,7 +384,10 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; + private final InformationGroup informationGroupMsg; + private final InformationTable topicsTable; + private final InformationAction msgButton; public MonitoringPage() { @@ -356,6 +408,17 @@ public MonitoringPage() { im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); + //TODO: rmv button + informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 2 ); + im.addGroup( informationGroupMsg ); + + msgButton = new InformationAction( informationGroupMsg, "Send a msg", (parameters) -> { + String end = "Msg was published!"; + client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); + return end; + }).withParameters( "topic", "msg" ); + im.registerInformation( msgButton ); + } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index c5ecae71f5..30edd004a4 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -23,6 +23,7 @@ import org.bson.BsonDocument; import org.bson.BsonInt32; import org.bson.BsonString; +import org.bson.codecs.pojo.annotations.BsonId; import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.DataStore; import org.polypheny.db.algebra.AlgNode; @@ -60,24 +61,28 @@ public class StreamCapture { StreamCapture( final TransactionManager transactionManager, PolyStream stream ) { this.transactionManager = transactionManager; this.stream = stream; - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - this.stream.setUserId( statement.getPrepareContext().getCurrentUserId() ); - this.stream.setDatabaseId( statement.getPrepareContext().getDatabaseId() ); + } public void handleContent() { - //String path = registerTopicFolder(topic); - long storeId = getCollectionID(); - if ( storeId != 0 ) { - stream.setStoreID( storeId ); - boolean saved = saveContent(); - //TODO: gescheite Tests + + if ( stream.getStoreID() == 0 ) { + //TODO: get store id of already existing or create new one. + + //TODO: maybe do this with interface + if ( stream.getNamespaceType() == NamespaceType.DOCUMENT ) { + long newstoreId = getCollectionID( ); + if ( newstoreId != 0 ) { + stream.setStoreID( newstoreId ); + } + } + } + boolean saved = saveContent(); + //TODO: gescheite Tests // Catalog catalog = Catalog.getInstance(); // CatalogSchema schema = null; // schema = catalog.getSchema( stream.getNamespaceID() ); - } } @@ -86,60 +91,22 @@ public void handleContent() { */ long getCollectionID() { Catalog catalog = Catalog.getInstance(); - List schemaList = catalog.getSchemas( this.stream.getDatabaseId(), null ); - - - // check for existing namespace with DOCUMENT NamespaceType: - if ( catalog.checkIfExistsSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ) ) { - - CatalogSchema schema = null; - try { - schema = catalog.getSchema( this.stream.getDatabaseId(), this.stream.getNamespace() ); - } catch ( UnknownSchemaException e ) { - log.error( "The catalog seems to be corrupt, as it was impossible to retrieve an existing namespace." ); - return 0; - } - - assert schema != null; - if ( schema.namespaceType == NamespaceType.DOCUMENT ) { - this.stream.setNamespaceID( schema.id ); - //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic - List collectionList = catalog.getCollections( schema.id, null ); - for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); - if ( !collection.placements.contains( queryInterfaceId ) ) { - return collection.addPlacement( queryInterfaceId ).id; - } else { - return collection.id; - } - } + //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic + List collectionList = catalog.getCollections( stream.getNamespaceID(), null ); + for ( CatalogCollection collection : collectionList ) { + if ( collection.name.equals( this.stream.topic ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + if ( !collection.placements.contains( queryInterfaceId ) ) { + return collection.addPlacement( queryInterfaceId ).id; + } else { + return collection.id; } - return createNewCollection(); - - } else { - this.stream.setNamespaceID( createNewNamespace() ); - return createNewCollection(); } - } else { - this.stream.setNamespaceID( createNewNamespace() ); - return createNewCollection(); } - } - + return createNewCollection(); - private long createNewNamespace() { - Catalog catalog = Catalog.getInstance(); - long namespaceId = catalog.addNamespace( stream.getNamespace(), stream.getDatabaseId(), stream.getUserId(), NamespaceType.DOCUMENT ); - try { - catalog.commit(); - } catch ( NoTablePrimaryKeyException e ) { - log.error( "An error " ); - } - return namespaceId; } - private long createNewCollection() { Catalog catalog = Catalog.getInstance(); @@ -159,10 +126,10 @@ private long createNewCollection() { log.info( "Created Collection with name: {}", this.stream.topic ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { - log.error( "The generation of the collection was not possible because there is a collaction already existing with this name." ); + log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); return 0; } catch ( TransactionException e ) { - log.error( "The commit after creating a new Collection could be completed!" ); + log.error( "The commit after creating a new Collection could not be completed!" ); return 0; } //add placement @@ -181,7 +148,7 @@ private long createNewCollection() { // added by Datomo public void insertDocument() { - String collectionName = "users"; + String collectionName = this.stream.topic; Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); @@ -190,8 +157,8 @@ public void insertDocument() { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); - document.put( "age", new BsonInt32( 28 ) ); - document.put( "name", new BsonString( "David" ) ); + document.put( "topic", new BsonString( this.stream.topic ) ); + document.put( "content", new BsonString( this.stream.getContent() ) ); AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); @@ -223,59 +190,9 @@ public void scanDocument() { boolean saveContent() { -/** - //TODO: save Message here -> Polyalgebra - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - AlgBuilder algBuilder = AlgBuilder.create( statement ); - JavaTypeFactory typeFactory = transaction.getTypeFactory(); - RexBuilder rexBuilder = new RexBuilder( typeFactory ); - - PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); - List names = new ArrayList<>(); - //TODO: change naming maybe - names.add( this.stream.topic ); - AlgOptTable table = catalogReader.getCollection( names ); - - // Values - AlgDataType tableRowType = table.getRowType( ); - List tableRows = tableRowType.getFieldList(); - - AlgOptPlanner planner = statement.getQueryProcessor().getPlanner(); - AlgOptCluster cluster = AlgOptCluster.create( planner, rexBuilder ); - - List valueColumnNames = this.valuesColumnNames( insertValueRequest.values ); - List rexValues = this.valuesNode( statement, algBuilder, rexBuilder, insertValueRequest, tableRows, inputStreams ).get( 0 ); - algBuilder.push( LogicalValues.createOneRow( cluster ) ); - algBuilder.project( rexValues, valueColumnNames ); - - // Table Modify - AlgNode algNode = algBuilder.build(); - Modify modify = new LogicalModify( - cluster, - algNode.getTraitSet(), - table, - catalogReader, - algNode, - LogicalModify.Operation.INSERT, - null, - null, - false - ); - - // Wrap {@link AlgNode} into a RelRoot - final AlgDataType rowType = modify.getRowType(); - final List> fields = Pair.zip( ImmutableIntList.identity( rowType.getFieldCount() ), rowType.getFieldNames() ); - final AlgCollation collation = - algNode instanceof Sort - ? ((Sort) algNode).collation - : AlgCollations.EMPTY; - AlgRoot root = new AlgRoot( modify, rowType, Kind.INSERT, fields, collation ); - log.debug( "AlgRoot was built." ); - - Context ctx = statement.getPrepareContext(); - log.info( executeAndTransformPolyAlg( root, statement, ctx ) ); - **/ + if ( this.stream.getNamespaceType() == NamespaceType.DOCUMENT ) { + insertDocument(); + } return true; } @@ -297,11 +214,6 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); log.debug( "AlgRoot was prepared." ); - /*final Iterable iterable = result.enumerable( statement.getDataContext() ); - Iterator iterator = iterable.iterator(); - while ( iterator.hasNext() ) { - iterator.next(); - }*/ // todo transform into desired output format List> rows = result.getRows( statement, -1 ); @@ -316,34 +228,6 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C } return null; } - //Pair result = restResult.getResult( ctx ); - - //return result.left; - //return null; - } - - - private static String registerTopicFolder( String topic ) { - PolyphenyHomeDirManager homeDirManager = PolyphenyHomeDirManager.getInstance(); - - String path = File.separator + "mqttStreamPlugin" + File.separator + topic.replace( "/", File.separator ); - ; - - File file = null; - if ( !homeDirManager.checkIfExists( path ) ) { - file = homeDirManager.registerNewFolder( path ); - log.info( "New Directory created!" ); - } else { - //TODO: rmv log - log.info( "Directory already exists" ); - } - - return path; - - } - - - public static void main( String[] args ) { } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java index 2d61e3effe..0a30cc16bb 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java @@ -22,7 +22,7 @@ @Slf4j public class StreamProcessing { - +//TODO: receive all additional info from Wrapper around MqttStream public static String processMsg( Mqtt3Publish subMsg ) { String msg = toString( subMsg ); From 7ba81bc22d498d47aab8cc7bb9ed00d1a90d6952 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 16 Jul 2023 13:10:22 +0200 Subject: [PATCH 039/114] Debugging of insertDocument ongoing --- .../org/polypheny/db/mqtt/StreamCapture.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 30edd004a4..05174f8d98 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -18,6 +18,7 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.bson.BsonDocument; @@ -26,9 +27,13 @@ import org.bson.codecs.pojo.annotations.BsonId; import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.algebra.AlgCollation; +import org.polypheny.db.algebra.AlgCollations; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.algebra.core.Sort; +import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; @@ -42,12 +47,17 @@ import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.plan.AlgOptTable; import org.polypheny.db.prepare.Context; +import org.polypheny.db.prepare.PolyphenyDbCatalogReader; +import org.polypheny.db.prepare.Prepare.PreparingTable; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; +import org.polypheny.db.util.ImmutableIntList; +import org.polypheny.db.util.Pair; import org.polypheny.db.util.PolyphenyHomeDirManager; @Slf4j @@ -157,13 +167,30 @@ public void insertDocument() { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); - document.put( "topic", new BsonString( this.stream.topic ) ); + //TODO: change to id: + document.put( "id", new BsonString( this.stream.topic ) ); document.put( "content", new BsonString( this.stream.getContent() ) ); AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); + //final AlgDataType rowType = algNode.getExpectedInputRowType(0); + List columnNumber = new ArrayList<>(); + columnNumber.add( 0, 1 ); + columnNumber.add( 1, 2 ); + + List columnNames = new ArrayList<>(); + columnNames.add( 0, "id" ); + columnNames.add( 1, "content" ); + + final List> fields = Pair.zip( columnNumber, columnNames ); + final AlgCollation collation = + algNode instanceof Sort + ? ((Sort) algNode).collation + : AlgCollations.EMPTY; + AlgRoot root = new AlgRoot( algNode, algNode.getRowType(), Kind.INSERT, fields, collation ); + // we can then wrap the tree in an AlgRoot and execute it - AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); + // AlgRoot root = AlgRoot.of( algNode, algNode.getRowType(), Kind.INSERT ); // for inserts and all DML queries only a number is returned String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); From 324409e54d0260fdceb63876d16d8596ca219eaf Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 16 Jul 2023 18:04:38 +0200 Subject: [PATCH 040/114] skeleton of StreamProcessing added --- .../polypheny/db/stream/StreamProcessor.java | 27 +++++++++ .../polypheny/db/transaction/Statement.java | 3 + .../db/stream/MqttStreamProcessor.java | 20 +++---- .../db/stream/StreamProcessorImpl.java | 55 +++++++++++++++++++ .../db/transaction/StatementImpl.java | 12 ++++ .../polypheny/db/mqtt/MqttStreamPlugin.java | 17 +++++- 6 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/org/polypheny/db/stream/StreamProcessor.java rename plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java => dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java (74%) create mode 100644 dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java diff --git a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java new file mode 100644 index 0000000000..5412cb959b --- /dev/null +++ b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.stream; + +/** + * + */ +public interface StreamProcessor { + + //TODO: maybe change type to MqttStream? + String processStream( String msg ); + +} diff --git a/core/src/main/java/org/polypheny/db/transaction/Statement.java b/core/src/main/java/org/polypheny/db/transaction/Statement.java index 7f88d9f958..d257ede19f 100644 --- a/core/src/main/java/org/polypheny/db/transaction/Statement.java +++ b/core/src/main/java/org/polypheny/db/transaction/Statement.java @@ -21,6 +21,7 @@ import org.polypheny.db.prepare.Context; import org.polypheny.db.monitoring.events.StatementEvent; import org.polypheny.db.processing.QueryProcessor; +import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.util.FileInputHandle; public interface Statement { @@ -29,6 +30,8 @@ public interface Statement { QueryProcessor getQueryProcessor(); + StreamProcessor getStreamProcessor(); + DataContext getDataContext(); Context getPrepareContext(); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java b/dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java similarity index 74% rename from plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java rename to dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java index 0a30cc16bb..14e711810a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamProcessing.java +++ b/dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java @@ -14,18 +14,21 @@ * limitations under the License. */ -package org.polypheny.db.mqtt; +package org.polypheny.db.stream; -import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.nio.charset.Charset; import lombok.extern.slf4j.Slf4j; @Slf4j -public class StreamProcessing { -//TODO: receive all additional info from Wrapper around MqttStream - public static String processMsg( Mqtt3Publish subMsg ) { - String msg = toString( subMsg ); +public class MqttStreamProcessor extends StreamProcessorImpl implements StreamProcessor{ + + public MqttStreamProcessor() { + } + + + //TODO: receive all additional info from Wrapper around MqttStream + public String processStream( String msg ) { String info = extractInfo( msg ); if ( validateMsg( info ) ) { log.info( "Extracted and validated message: {}", msg); @@ -37,11 +40,6 @@ public static String processMsg( Mqtt3Publish subMsg ) { } - private static String toString( Mqtt3Publish subMsg ) { - return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); - } - - private static boolean validateMsg( String msg ) { //TODO: Implement return true; diff --git a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java new file mode 100644 index 0000000000..ec5d732bd8 --- /dev/null +++ b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.stream; + +import static org.reflections.Reflections.log; + +public class StreamProcessorImpl implements StreamProcessor { + + public StreamProcessorImpl() { + + } + + + @Override + //TODO: receive all additional info from Wrapper around MqttStream + public String processStream( String msg ) { + String info = extractInfo( msg ); + if ( validateMsg( info ) ) { + log.info( "Extracted and validated message: {}", msg); + return info; + } else { + log.error( "Message is not valid!" ); + return null; + } + } + + + private static boolean validateMsg( String msg ) { + //TODO: Implement + return true; + } + + + private static String extractInfo( String msg ) { + //TODO: extract the needed Info only -> based on topic attribut on right side!!}; + return msg; + } + + + +} diff --git a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java index 4ff6444b05..9cc5e7b87b 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java @@ -35,6 +35,9 @@ import org.polypheny.db.processing.QueryProcessor; import org.polypheny.db.processing.QueryProviderImpl; import org.polypheny.db.processing.VolcanoQueryProcessor; +import org.polypheny.db.stream.StreamProcessor; +import org.polypheny.db.stream.MqttStreamProcessor; +import org.polypheny.db.stream.StreamProcessorImpl; import org.polypheny.db.util.FileInputHandle; public class StatementImpl implements Statement { @@ -73,6 +76,15 @@ public QueryProcessor getQueryProcessor() { } + @Override + public StreamProcessor getStreamProcessor() { + /**if ( streamProcessor == null ) { + + }**/ + return new MqttStreamProcessor(); + } + + @Override public DataContext getDataContext() { if ( dataContext == null ) { diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 1c19e3347f..bdb75534c3 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -21,6 +21,7 @@ import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -45,6 +46,7 @@ import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; +import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; @@ -340,7 +342,15 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. - String content = StreamProcessing.processMsg( subMsg ); + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + StreamProcessor streamProcessor = statement.getStreamProcessor(); + + + String content = streamProcessor.processStream(extractPayload(subMsg)); + + + PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); stream.setNamespaceID( this.namespaceId ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); @@ -348,6 +358,11 @@ void processMsg( Mqtt3Publish subMsg ) { } + private static String extractPayload( Mqtt3Publish subMsg ) { + return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); + } + + public List topicsToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { From 608e7c6ec82cbdbbac089ace27dccd44a8a2e48d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 10:28:58 +0200 Subject: [PATCH 041/114] fixed issue with insertDocument inserting documents works now! --- .../src/main/java/org/polypheny/db/tools/AlgBuilder.java | 6 ++++++ .../main/java/org/polypheny/db/mqtt/StreamCapture.java | 9 +++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java b/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java index 7a43dd1fac..94b81a83b2 100644 --- a/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java +++ b/core/src/main/java/org/polypheny/db/tools/AlgBuilder.java @@ -297,6 +297,12 @@ public static AlgBuilder create( Statement statement ) { return create( statement, cluster ); } + public static AlgBuilder createDocumentBuilder( Statement statement ) { + final RexBuilder rexBuilder = new RexBuilder( statement.getTransaction().getTypeFactory() ); + final AlgOptCluster cluster = AlgOptCluster.createDocument( statement.getQueryProcessor().getPlanner(), rexBuilder ); + return create( statement, cluster ); + } + public static AlgBuilder create( Statement statement, AlgOptCluster cluster ) { return new AlgBuilder( Contexts.EMPTY_CONTEXT, cluster, statement.getTransaction().getCatalogReader() ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 05174f8d98..8599f68393 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -47,6 +47,7 @@ import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.plan.AlgOptCluster; import org.polypheny.db.plan.AlgOptTable; import org.polypheny.db.prepare.Context; import org.polypheny.db.prepare.PolyphenyDbCatalogReader; @@ -158,17 +159,17 @@ private long createNewCollection() { // added by Datomo public void insertDocument() { - String collectionName = this.stream.topic; + String collectionName = "wohnzimmer." + this.stream.topic; Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed - AlgBuilder builder = AlgBuilder.create( statement ); + AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); //TODO: change to id: - document.put( "id", new BsonString( this.stream.topic ) ); + document.put( "topic", new BsonString( this.stream.topic ) ); document.put( "content", new BsonString( this.stream.getContent() ) ); AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); @@ -247,7 +248,7 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C statement.getTransaction().commit(); return rows.toString(); } catch ( Throwable e ) { - log.error( "Error during execution of REST query", e ); + log.error( "Error during execution of stream capture query", e ); try { statement.getTransaction().rollback(); } catch ( TransactionException transactionException ) { From d7d94e2f18753405a36fc202fa5863e2cf6ee7e9 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 10:29:51 +0200 Subject: [PATCH 042/114] changed Module of MqttStreamProcessor --- .../src/main/java/org/polypheny/db/mqtt}/MqttStreamProcessor.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {dbms/src/main/java/org/polypheny/db/stream => plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt}/MqttStreamProcessor.java (100%) diff --git a/dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java similarity index 100% rename from dbms/src/main/java/org/polypheny/db/stream/MqttStreamProcessor.java rename to plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java From 75956c0ebc0fa7522ec56553144fc3c7047fd1d9 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 11:53:43 +0200 Subject: [PATCH 043/114] added StreamMessage to module stream incl. mqtt implementation: MqttMessage. incl. Wrapper for MqttMessage: ReceivedMqttMessage and PublishingMessage. --- .../polypheny/db/stream/StreamMessage.java | 25 ++++++++ .../org/polypheny/db/mqtt/MqttMessage.java | 34 +++++++++++ .../db/mqtt/PublishingMqttMessage.java | 23 +++++++ .../db/mqtt/ReceivedMqttMessage.java | 60 +++++++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 core/src/main/java/org/polypheny/db/stream/StreamMessage.java create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java create mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java diff --git a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java new file mode 100644 index 0000000000..0356beb863 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.stream; + +import lombok.Getter; + +public interface StreamMessage { + + + +} diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java new file mode 100644 index 0000000000..e910e25594 --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import lombok.Getter; +import org.polypheny.db.stream.StreamMessage; + +public class MqttMessage implements StreamMessage { + @Getter + final String message; + @Getter + final String topic; + + + public MqttMessage( String message, String topic ) { + this.message = message; + this.topic = topic; + } + +} diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java new file mode 100644 index 0000000000..e35c47d97a --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +public class PublishingMqttMessage { + + MqttMessage msg; + +} diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java new file mode 100644 index 0000000000..3daf4ae42a --- /dev/null +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import lombok.Getter; +import lombok.Setter; +import org.polypheny.db.catalog.Catalog.NamespaceType; + +public class ReceivedMqttMessage { + private final MqttMessage msg; + @Getter + private final String namespaceName; + @Getter + private final long namespaceId; + @Getter + private final NamespaceType namespaceType; + @Getter + private final String uniqueNameOfInterface; + @Getter + private final long databaseId; + @Getter + private final int userId; + @Getter @Setter + private long storeId; + + + public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, long storeId, String uniqueNameOfInterface, long databaseId, int userId ) { + this.msg = msg; + this.namespaceName = namespaceName; + this.namespaceId = namespaceId; + this.namespaceType = namespaceType; + this.uniqueNameOfInterface = uniqueNameOfInterface; + this.databaseId = databaseId; + this.userId = userId; + this.storeId = storeId; + } + + public String getTopic() { + return this.msg.getTopic(); + } + + public String getMessage() { + return this.msg.getMessage(); + } + +} From 8220800554b6a8fdb2a54bf3dea65e64773cb240 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 11:54:37 +0200 Subject: [PATCH 044/114] added getStreamProcessor method --- .../main/java/org/polypheny/db/transaction/StatementImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java index 9cc5e7b87b..ee9823f3fe 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java @@ -36,7 +36,6 @@ import org.polypheny.db.processing.QueryProviderImpl; import org.polypheny.db.processing.VolcanoQueryProcessor; import org.polypheny.db.stream.StreamProcessor; -import org.polypheny.db.stream.MqttStreamProcessor; import org.polypheny.db.stream.StreamProcessorImpl; import org.polypheny.db.util.FileInputHandle; @@ -81,7 +80,7 @@ public StreamProcessor getStreamProcessor() { /**if ( streamProcessor == null ) { }**/ - return new MqttStreamProcessor(); + return new StreamProcessorImpl(); } From 7dedc4fb86dfc79f2c1d96beb8edad7d3fb12025 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 11:56:29 +0200 Subject: [PATCH 045/114] integrated MqttMessage objects in existing code --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 12 ++-- .../org/polypheny/db/mqtt/StreamCapture.java | 56 ++++++++----------- 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index bdb75534c3..b546b5b9c3 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -344,16 +344,14 @@ void processMsg( Mqtt3Publish subMsg ) { //TODO: attention: return values, not correct, might need a change of type. Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - StreamProcessor streamProcessor = statement.getStreamProcessor(); - - - String content = streamProcessor.processStream(extractPayload(subMsg)); + ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage(new MqttMessage( extractPayload(subMsg), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, getUniqueName(), this.databaseId, this.userId ); + StreamProcessor streamProcessor = statement.getStreamProcessor(); + //TODO: ganzes ReceivedMqttMEssage objekt übergeben und arbeiten lassen. + String content = streamProcessor.processStream(receivedMqttMessage.getMessage()); - PolyStream stream = new PolyStream( subMsg.getTopic().toString(), getUniqueName(), content, this.namespace, this.namespaceType ); - stream.setNamespaceID( this.namespaceId ); - StreamCapture streamCapture = new StreamCapture( this.transactionManager, stream ); + StreamCapture streamCapture = new StreamCapture( this.transactionManager, receivedMqttMessage ); streamCapture.handleContent(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 8599f68393..3ddf3bb531 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -16,15 +16,11 @@ package org.polypheny.db.mqtt; -import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.bson.BsonDocument; -import org.bson.BsonInt32; import org.bson.BsonString; -import org.bson.codecs.pojo.annotations.BsonId; import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.DataStore; import org.polypheny.db.algebra.AlgCollation; @@ -33,31 +29,23 @@ import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.algebra.core.Sort; -import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; -import org.polypheny.db.catalog.entity.CatalogSchema; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; import org.polypheny.db.catalog.exceptions.GenericCatalogException; -import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; -import org.polypheny.db.plan.AlgOptCluster; -import org.polypheny.db.plan.AlgOptTable; import org.polypheny.db.prepare.Context; -import org.polypheny.db.prepare.PolyphenyDbCatalogReader; -import org.polypheny.db.prepare.Prepare.PreparingTable; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; -import org.polypheny.db.util.ImmutableIntList; import org.polypheny.db.util.Pair; import org.polypheny.db.util.PolyphenyHomeDirManager; @@ -66,26 +54,26 @@ public class StreamCapture { TransactionManager transactionManager; PolyphenyHomeDirManager homeDirManager; - PolyStream stream; + ReceivedMqttMessage receivedMqttMessage; - StreamCapture( final TransactionManager transactionManager, PolyStream stream ) { + StreamCapture( final TransactionManager transactionManager, ReceivedMqttMessage receivedMqttMessage ) { this.transactionManager = transactionManager; - this.stream = stream; + this.receivedMqttMessage = receivedMqttMessage; } public void handleContent() { - if ( stream.getStoreID() == 0 ) { + if ( receivedMqttMessage.getStoreId() == 0 ) { //TODO: get store id of already existing or create new one. //TODO: maybe do this with interface - if ( stream.getNamespaceType() == NamespaceType.DOCUMENT ) { - long newstoreId = getCollectionID( ); + if ( receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { + long newstoreId = getCollectionId(); if ( newstoreId != 0 ) { - stream.setStoreID( newstoreId ); + receivedMqttMessage.setStoreId( newstoreId ); } } } @@ -100,13 +88,13 @@ public void handleContent() { /** * @return the id of the collection that was either already existing with the topic as name or that was newly created */ - long getCollectionID() { + long getCollectionId() { Catalog catalog = Catalog.getInstance(); //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic - List collectionList = catalog.getCollections( stream.getNamespaceID(), null ); + List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); if ( !collection.placements.contains( queryInterfaceId ) ) { return collection.addPlacement( queryInterfaceId ).id; } else { @@ -128,13 +116,13 @@ private long createNewCollection() { try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( - this.stream.getNamespaceID(), - this.stream.topic, + this.receivedMqttMessage.getNamespaceId(), + this.receivedMqttMessage.getTopic(), true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created Collection with name: {}", this.stream.topic ); + log.info( "Created Collection with name: {}", this.receivedMqttMessage.getTopic() ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); @@ -144,10 +132,10 @@ private long createNewCollection() { return 0; } //add placement - List collectionList = catalog.getCollections( this.stream.getNamespaceID(), null ); + List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); for ( int i = 0; i < collectionList.size(); i++ ) { - if ( collectionList.get( i ).name.equals( this.stream.topic ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.stream.getUniqueNameOfInterface() ).getQueryInterfaceId(); + if ( collectionList.get( i ).name.equals( this.receivedMqttMessage.getTopic() ) ) { + int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); collectionList.set( i, collectionList.get( i ).addPlacement( queryInterfaceId ) ); return collectionList.get( i ).id; @@ -159,7 +147,7 @@ private long createNewCollection() { // added by Datomo public void insertDocument() { - String collectionName = "wohnzimmer." + this.stream.topic; + String collectionName = "wohnzimmer." + this.receivedMqttMessage.getTopic(); Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); @@ -169,8 +157,8 @@ public void insertDocument() { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); //TODO: change to id: - document.put( "topic", new BsonString( this.stream.topic ) ); - document.put( "content", new BsonString( this.stream.getContent() ) ); + document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); + document.put( "content", new BsonString( this.receivedMqttMessage.getMessage() ) ); AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); @@ -218,7 +206,7 @@ public void scanDocument() { boolean saveContent() { - if ( this.stream.getNamespaceType() == NamespaceType.DOCUMENT ) { + if ( this.receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { insertDocument(); } return true; @@ -227,7 +215,7 @@ boolean saveContent() { private Transaction getTransaction() { try { - return transactionManager.startTransaction( this.stream.getUserId(), this.stream.getDatabaseId(), false, "MQTT Stream" ); + return transactionManager.startTransaction( this.receivedMqttMessage.getUserId(), this.receivedMqttMessage.getDatabaseId(), false, "MQTT Stream" ); } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { throw new RuntimeException( "Error while starting transaction", e ); } From ff33451ae12539a93070ab5a381668fe890a271d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 12:05:42 +0200 Subject: [PATCH 046/114] Reformatted files --- .../polypheny/db/stream/StreamMessage.java | 5 +---- .../org/polypheny/db/mqtt/MqttMessage.java | 1 + .../polypheny/db/mqtt/MqttStreamPlugin.java | 20 +++++++++---------- .../db/mqtt/ReceivedMqttMessage.java | 8 ++++++-- .../org/polypheny/db/mqtt/StreamCapture.java | 3 ++- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java index 0356beb863..26e9995f0c 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java @@ -16,10 +16,7 @@ package org.polypheny.db.stream; -import lombok.Getter; - -public interface StreamMessage { - +public interface StreamMessage { } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java index e910e25594..7462e86bb0 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java @@ -20,6 +20,7 @@ import org.polypheny.db.stream.StreamMessage; public class MqttMessage implements StreamMessage { + @Getter final String message; @Getter diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index b546b5b9c3..81cdd06299 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -137,11 +137,11 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.databaseId = statement.getPrepareContext().getDatabaseId(); String name = settings.get( "namespace" ); - NamespaceType type = NamespaceType.valueOf(settings.get( "namespace type" )); + NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); long namespaceId = getNamespaceId( name, type ); if ( namespaceId != 0 ) { this.namespace = name; - this.namespaceType = type; + this.namespaceType = type; this.namespaceId = namespaceId; } @@ -216,7 +216,7 @@ private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) try { schema = catalog.getSchema( this.databaseId, namespaceName ); } catch ( UnknownSchemaException e ) { - throw new RuntimeException(e); + throw new RuntimeException( e ); } assert schema != null; if ( schema.namespaceType == namespaceType ) { @@ -232,7 +232,7 @@ private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) catalog.commit(); namespaceId = id; } catch ( NoTablePrimaryKeyException e ) { - throw new RuntimeException(e); + throw new RuntimeException( e ); } } return namespaceId; @@ -271,7 +271,7 @@ protected void reloadSettings( List updatedSettings ) { case "namespace": String newNamespaceName = this.getCurrentSettings().get( "namespace" ); - long namespaceId1 = this.getNamespaceId( newNamespaceName, this.namespaceType); + long namespaceId1 = this.getNamespaceId( newNamespaceName, this.namespaceType ); if ( namespaceId1 != 0 ) { this.namespaceId = namespaceId1; this.namespace = newNamespaceName; @@ -279,7 +279,7 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - long namespaceId2 = this.getNamespaceId( this.namespace, newNamespaceType); + long namespaceId2 = this.getNamespaceId( this.namespace, newNamespaceType ); if ( namespaceId2 != 0 ) { this.namespaceId = namespaceId2; this.namespaceType = newNamespaceType; @@ -345,11 +345,11 @@ void processMsg( Mqtt3Publish subMsg ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage(new MqttMessage( extractPayload(subMsg), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, getUniqueName(), this.databaseId, this.userId ); + ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId ); StreamProcessor streamProcessor = statement.getStreamProcessor(); //TODO: ganzes ReceivedMqttMEssage objekt übergeben und arbeiten lassen. - String content = streamProcessor.processStream(receivedMqttMessage.getMessage()); + String content = streamProcessor.processStream( receivedMqttMessage.getMessage() ); StreamCapture streamCapture = new StreamCapture( this.transactionManager, receivedMqttMessage ); streamCapture.handleContent(); @@ -425,11 +425,11 @@ public MonitoringPage() { informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 2 ); im.addGroup( informationGroupMsg ); - msgButton = new InformationAction( informationGroupMsg, "Send a msg", (parameters) -> { + msgButton = new InformationAction( informationGroupMsg, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); return end; - }).withParameters( "topic", "msg" ); + } ).withParameters( "topic", "msg" ); im.registerInformation( msgButton ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java index 3daf4ae42a..84eb2d97bf 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java @@ -21,6 +21,7 @@ import org.polypheny.db.catalog.Catalog.NamespaceType; public class ReceivedMqttMessage { + private final MqttMessage msg; @Getter private final String namespaceName; @@ -34,8 +35,9 @@ public class ReceivedMqttMessage { private final long databaseId; @Getter private final int userId; - @Getter @Setter - private long storeId; + @Getter + @Setter + private long storeId; // the ID of collection/graph/table... the place where info is/should be saved public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, long storeId, String uniqueNameOfInterface, long databaseId, int userId ) { @@ -49,10 +51,12 @@ public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespac this.storeId = storeId; } + public String getTopic() { return this.msg.getTopic(); } + public String getMessage() { return this.msg.getMessage(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 3ddf3bb531..b1c4077804 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -90,7 +90,7 @@ public void handleContent() { */ long getCollectionId() { Catalog catalog = Catalog.getInstance(); - //check for collection with same name //TODO: maybe change the collection name, currently collection name is the topic + //check for collection with same name List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); for ( CatalogCollection collection : collectionList ) { if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { @@ -106,6 +106,7 @@ long getCollectionId() { } + private long createNewCollection() { Catalog catalog = Catalog.getInstance(); From 5a01216a57baac56fe8caefc8ae06fc8a92f5bde Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 12:06:11 +0200 Subject: [PATCH 047/114] added EntityType STREAM --- core/src/main/java/org/polypheny/db/catalog/Catalog.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/catalog/Catalog.java b/core/src/main/java/org/polypheny/db/catalog/Catalog.java index 3bb729294f..44f984eec4 100644 --- a/core/src/main/java/org/polypheny/db/catalog/Catalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/Catalog.java @@ -1933,8 +1933,8 @@ public enum EntityType { ENTITY( 1 ), SOURCE( 2 ), VIEW( 3 ), - MATERIALIZED_VIEW( 4 ); - // STREAM, ... + MATERIALIZED_VIEW( 4 ), + STREAM(5); private final int id; From 2c0113df1c1fd32622d7db68d858afcb973aceef Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 17 Jul 2023 13:36:37 +0200 Subject: [PATCH 048/114] integrated MqttMessage objects in MqttStreamProcessor --- .../polypheny/db/mqtt/MqttStreamProcessor.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index 14e711810a..cb80e494e6 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -14,22 +14,24 @@ * limitations under the License. */ -package org.polypheny.db.stream; +package org.polypheny.db.mqtt; import java.nio.charset.Charset; import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.mqtt.ReceivedMqttMessage; @Slf4j -public class MqttStreamProcessor extends StreamProcessorImpl implements StreamProcessor{ +public class MqttStreamProcessor { + ReceivedMqttMessage receivedMqttMessage; - - public MqttStreamProcessor() { + public MqttStreamProcessor(ReceivedMqttMessage receivedMqttMessage) { + this.receivedMqttMessage = receivedMqttMessage; } //TODO: receive all additional info from Wrapper around MqttStream - public String processStream( String msg ) { - String info = extractInfo( msg ); + public String processStream( ReceivedMqttMessage msg ) { + String info = extractInfo( msg.getMessage(), msg.getTopic() ); if ( validateMsg( info ) ) { log.info( "Extracted and validated message: {}", msg); return info; @@ -46,7 +48,7 @@ private static boolean validateMsg( String msg ) { } - private static String extractInfo( String msg ) { + private static String extractInfo( String msg, String topic ) { //TODO: extract the needed Info only -> based on topic attribut on right side!!}; return msg; } From 0cc5df64935c86166beab592c1bd810c64c78867 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 18 Jul 2023 13:37:54 +0200 Subject: [PATCH 049/114] changed constructor parameter from TransactionManager to Transaction --- .../org/polypheny/db/mqtt/StreamCapture.java | 48 ++++++------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index b1c4077804..569de07803 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -52,13 +52,13 @@ @Slf4j public class StreamCapture { - TransactionManager transactionManager; + Transaction transaction; PolyphenyHomeDirManager homeDirManager; ReceivedMqttMessage receivedMqttMessage; - StreamCapture( final TransactionManager transactionManager, ReceivedMqttMessage receivedMqttMessage ) { - this.transactionManager = transactionManager; + StreamCapture( final Transaction transaction, ReceivedMqttMessage receivedMqttMessage ) { + this.transaction = transaction; this.receivedMqttMessage = receivedMqttMessage; } @@ -96,8 +96,10 @@ long getCollectionId() { if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); if ( !collection.placements.contains( queryInterfaceId ) ) { + log.info( "found matching collection!" ); return collection.addPlacement( queryInterfaceId ).id; } else { + log.info( "found matching collection!" ); return collection.id; } } @@ -111,7 +113,6 @@ private long createNewCollection() { Catalog catalog = Catalog.getInstance(); //Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; - Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); try { @@ -123,7 +124,7 @@ private long createNewCollection() { dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created Collection with name: {}", this.receivedMqttMessage.getTopic() ); + log.info( "Created new collection with name: {}", this.receivedMqttMessage.getTopic() ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); @@ -133,6 +134,7 @@ private long createNewCollection() { return 0; } //add placement + //TODO:insert Placements permanently: currently new placement is only inserted locally!!!! List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); for ( int i = 0; i < collectionList.size(); i++ ) { if ( collectionList.get( i ).name.equals( this.receivedMqttMessage.getTopic() ) ) { @@ -149,7 +151,6 @@ private long createNewCollection() { // added by Datomo public void insertDocument() { String collectionName = "wohnzimmer." + this.receivedMqttMessage.getTopic(); - Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -163,27 +164,16 @@ public void insertDocument() { AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); - //final AlgDataType rowType = algNode.getExpectedInputRowType(0); - List columnNumber = new ArrayList<>(); - columnNumber.add( 0, 1 ); - columnNumber.add( 1, 2 ); - - List columnNames = new ArrayList<>(); - columnNames.add( 0, "id" ); - columnNames.add( 1, "content" ); - - final List> fields = Pair.zip( columnNumber, columnNames ); - final AlgCollation collation = - algNode instanceof Sort - ? ((Sort) algNode).collation - : AlgCollations.EMPTY; - AlgRoot root = new AlgRoot( algNode, algNode.getRowType(), Kind.INSERT, fields, collation ); - // we can then wrap the tree in an AlgRoot and execute it - // AlgRoot root = AlgRoot.of( algNode, algNode.getRowType(), Kind.INSERT ); + AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); // for inserts and all DML queries only a number is returned String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); - + try { + transaction.commit(); + } catch ( TransactionException e ) { + throw new RuntimeException( e ); + } + log.info( "inserted message as document" ); } @@ -191,7 +181,6 @@ public void insertDocument() { // added by Datomo public void scanDocument() { String collectionName = "users"; - Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -214,15 +203,6 @@ boolean saveContent() { } - private Transaction getTransaction() { - try { - return transactionManager.startTransaction( this.receivedMqttMessage.getUserId(), this.receivedMqttMessage.getDatabaseId(), false, "MQTT Stream" ); - } catch ( UnknownUserException | UnknownDatabaseException | UnknownSchemaException | GenericCatalogException e ) { - throw new RuntimeException( "Error while starting transaction", e ); - } - } - - String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { //TODO: implement From e1bd4012bc7b488cfbb286e7b59940840eb234f1 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 18 Jul 2023 13:41:02 +0200 Subject: [PATCH 050/114] change of StreamProcessorImpl constructor --- .../polypheny/db/transaction/Statement.java | 2 +- .../db/stream/StreamProcessorImpl.java | 31 +++---------------- .../db/transaction/StatementImpl.java | 7 ++--- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/transaction/Statement.java b/core/src/main/java/org/polypheny/db/transaction/Statement.java index d257ede19f..3704b02c01 100644 --- a/core/src/main/java/org/polypheny/db/transaction/Statement.java +++ b/core/src/main/java/org/polypheny/db/transaction/Statement.java @@ -30,7 +30,7 @@ public interface Statement { QueryProcessor getQueryProcessor(); - StreamProcessor getStreamProcessor(); + StreamProcessor getStreamProcessor(String message); DataContext getDataContext(); diff --git a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java index ec5d732bd8..25805f292b 100644 --- a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java @@ -20,36 +20,15 @@ public class StreamProcessorImpl implements StreamProcessor { - public StreamProcessorImpl() { - + String stream; + public StreamProcessorImpl(String stream) { + this.stream = stream; } @Override - //TODO: receive all additional info from Wrapper around MqttStream - public String processStream( String msg ) { - String info = extractInfo( msg ); - if ( validateMsg( info ) ) { - log.info( "Extracted and validated message: {}", msg); - return info; - } else { - log.error( "Message is not valid!" ); - return null; - } + public String getStream() { + return stream; } - - private static boolean validateMsg( String msg ) { - //TODO: Implement - return true; - } - - - private static String extractInfo( String msg ) { - //TODO: extract the needed Info only -> based on topic attribut on right side!!}; - return msg; - } - - - } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java index ee9823f3fe..50b8650d61 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/StatementImpl.java @@ -76,11 +76,8 @@ public QueryProcessor getQueryProcessor() { @Override - public StreamProcessor getStreamProcessor() { - /**if ( streamProcessor == null ) { - - }**/ - return new StreamProcessorImpl(); + public StreamProcessor getStreamProcessor(String message) { + return new StreamProcessorImpl(message); } From cd72d1c0f2ebbbc71e0dade7fcedf3d6a3eeff62 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 18 Jul 2023 14:04:20 +0200 Subject: [PATCH 051/114] fixed accessing StreamProcessorImpl from MqttStreamProcessor --- .../polypheny/db/stream/StreamProcessor.java | 2 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 23 +++++++++++------- .../db/mqtt/MqttStreamProcessor.java | 24 ++++++++++++++----- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java index 5412cb959b..e81bdd6d1e 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java @@ -22,6 +22,6 @@ public interface StreamProcessor { //TODO: maybe change type to MqttStream? - String processStream( String msg ); + String getStream( ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 81cdd06299..e75d27aef3 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -46,6 +46,7 @@ import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; +import org.polypheny.db.information.InformationText; import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; @@ -346,12 +347,10 @@ void processMsg( Mqtt3Publish subMsg ) { Statement statement = transaction.createStatement(); ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); + String content = streamProcessor.processStream(); - StreamProcessor streamProcessor = statement.getStreamProcessor(); - //TODO: ganzes ReceivedMqttMEssage objekt übergeben und arbeiten lassen. - String content = streamProcessor.processStream( receivedMqttMessage.getMessage() ); - - StreamCapture streamCapture = new StreamCapture( this.transactionManager, receivedMqttMessage ); + StreamCapture streamCapture = new StreamCapture( getTransaction(), receivedMqttMessage ); streamCapture.handleContent(); } @@ -398,7 +397,7 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; private final InformationGroup informationGroupMsg; - + private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; private final InformationAction msgButton; @@ -407,7 +406,15 @@ public MonitoringPage() { InformationManager im = InformationManager.getInstance(); informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).fullWidth().setLabel( "Interfaces" ); - informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 1 ); + + informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); + im.addGroup( informationGroupInfo ); + InformationText brokerInfo = new InformationText( informationGroupInfo, "Broker IP: " + broker + ":" + brokerPort ); + InformationText namespaceName = new InformationText( informationGroupInfo,"Namespace name: " + namespace); + im.registerInformation( brokerInfo ); + im.registerInformation( namespaceName ); + + informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); im.addPage( informationPage ); im.addGroup( informationGroupTopics ); @@ -422,7 +429,7 @@ public MonitoringPage() { informationGroupTopics.setRefreshFunction( this::update ); //TODO: rmv button - informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 2 ); + informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupMsg ); msgButton = new InformationAction( informationGroupMsg, "Send a msg", ( parameters ) -> { diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index cb80e494e6..887aa5140d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -19,21 +19,28 @@ import java.nio.charset.Charset; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.mqtt.ReceivedMqttMessage; +import org.polypheny.db.stream.StreamProcessor; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; @Slf4j -public class MqttStreamProcessor { - ReceivedMqttMessage receivedMqttMessage; +public class MqttStreamProcessor implements StreamProcessor { - public MqttStreamProcessor(ReceivedMqttMessage receivedMqttMessage) { + private final ReceivedMqttMessage receivedMqttMessage; + private final StreamProcessor streamProcessor; + + public MqttStreamProcessor(ReceivedMqttMessage receivedMqttMessage, Statement statement ) { this.receivedMqttMessage = receivedMqttMessage; + this.streamProcessor = statement.getStreamProcessor( receivedMqttMessage.getMessage() ); + } //TODO: receive all additional info from Wrapper around MqttStream - public String processStream( ReceivedMqttMessage msg ) { - String info = extractInfo( msg.getMessage(), msg.getTopic() ); + public String processStream( ) { + String info = extractInfo( this.receivedMqttMessage.getMessage(), this.receivedMqttMessage.getTopic() ); if ( validateMsg( info ) ) { - log.info( "Extracted and validated message: {}", msg); + log.info( "Extracted and validated message: {}", this.receivedMqttMessage.getMessage()); return info; } else { log.error( "Message is not valid!" ); @@ -54,4 +61,9 @@ private static String extractInfo( String msg, String topic ) { } + @Override + public String getStream() { + return streamProcessor.getStream(); + } + } From 71bd1d79343cf9325c49126c676bb76bf5f2f4db Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 18 Jul 2023 17:07:40 +0200 Subject: [PATCH 052/114] changed topics List to a ConcurrentHashMap also integrated it everywhere, also in UI --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index e75d27aef3..adf60df9de 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -26,6 +26,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.Plugin; @@ -101,13 +106,14 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingString( "topics", false, true, true, null ) ); + @Getter private final String broker; - + @Getter private final int brokerPort; - private ArrayList topics = new ArrayList(); + private Map topics = new ConcurrentHashMap<>(); - private static Mqtt3AsyncClient client; + private Mqtt3AsyncClient client; private String namespace; @@ -249,7 +255,7 @@ protected void reloadSettings( List updatedSettings ) { List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { - if ( !topics.contains( newTopic ) ) { + if ( !topics.containsKey( newTopic ) ) { topicsToSub.add( newTopic ); } } @@ -259,7 +265,7 @@ protected void reloadSettings( List updatedSettings ) { } List topicsToUnsub = new ArrayList<>(); - for ( String oldTopic : topics ) { + for ( String oldTopic : topics.keySet() ) { if ( !newTopicsList.contains( oldTopic ) ) { topicsToUnsub.add( oldTopic ); } @@ -312,7 +318,7 @@ public void subscribe( String topic ) { if ( throwable != null ) { log.info( "Subscription was not successfull. Please try again." ); } else { - this.topics.add( topic ); + this.topics.put( topic, new AtomicLong(0)); log.info( "Successful subscription to topic:{}.", topic ); } } ); @@ -342,16 +348,20 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { - //TODO: attention: return values, not correct, might need a change of type. Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); - + //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? + if ( !Objects.equals( content, "" ) ) { + Long incrementedCount = topics.get( receivedMqttMessage.getTopic() ).incrementAndGet(); + topics.replace( receivedMqttMessage.getTopic(), new AtomicLong( incrementedCount ) ); + } StreamCapture streamCapture = new StreamCapture( getTransaction(), receivedMqttMessage ); streamCapture.handleContent(); + this.monitoringPage.update(); } @@ -400,6 +410,10 @@ private class MonitoringPage { private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; private final InformationAction msgButton; + private final InformationText brokerInfo; + private final InformationText clientInfo; + private final InformationText namespaceName; + public MonitoringPage() { @@ -409,9 +423,12 @@ public MonitoringPage() { informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); - InformationText brokerInfo = new InformationText( informationGroupInfo, "Broker IP: " + broker + ":" + brokerPort ); - InformationText namespaceName = new InformationText( informationGroupInfo,"Namespace name: " + namespace); + + brokerInfo = new InformationText( informationGroupInfo, "0.0.0.0:0000" ); im.registerInformation( brokerInfo ); + clientInfo = new InformationText( informationGroupInfo, "o" ); + im.registerInformation( clientInfo ); + namespaceName = new InformationText( informationGroupInfo, "namespace" ); im.registerInformation( namespaceName ); informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); @@ -422,7 +439,7 @@ public MonitoringPage() { // table to display topics topicsTable = new InformationTable( informationGroupTopics, - List.of( "Topics" ) + List.of( "Topic", "Number of received messages" ) ); im.registerInformation( topicsTable ); @@ -447,10 +464,18 @@ public void update() { if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - for ( String topic : topics ) { - topicsTable.addRow( topic ); + for ( String topic : topics.keySet() ) { + //TODO: format + topicsTable.addRow( topic, topics.get( topic ) ); } } + this.brokerInfo.setText( "Broker address: " + client.getConfig().getServerHost() + + "\n" + "Broker port:" + client.getConfig().getServerPort() + + "\n Broker version of MQTT : " + String.valueOf( client.getConfig().getMqttVersion() ) + + "\n" + "SSL configuration: " + client.getConfig().getSslConfig() ); + //TODO: check this after having SSL Configuration. + this.clientInfo.setText( "Client state: " + String.valueOf( client.getState() ) ); + this.namespaceName.setText( "Namespace name: " + namespace ); } From 81b13a8db79f7c93a639cebab2206a56f74e4070 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Wed, 19 Jul 2023 20:12:50 +0200 Subject: [PATCH 053/114] Monitoring Page now shows recent messages --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 21 ++++-- .../org/polypheny/db/mqtt/StreamCapture.java | 75 +++++++++++-------- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index adf60df9de..3730f4b83a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -186,7 +186,6 @@ public void run() { } } ); - } @@ -231,6 +230,7 @@ private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) } else { log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); + throw new RuntimeException(); } } else { @@ -341,6 +341,7 @@ public void unsubscribe( String topic ) { log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { this.topics.remove( topic ); + //TODO: remove collction in StreamCapture log.info( "Unsubscribed from topic:{}.", topic ); } } ); @@ -359,8 +360,8 @@ void processMsg( Mqtt3Publish subMsg ) { Long incrementedCount = topics.get( receivedMqttMessage.getTopic() ).incrementAndGet(); topics.replace( receivedMqttMessage.getTopic(), new AtomicLong( incrementedCount ) ); } - StreamCapture streamCapture = new StreamCapture( getTransaction(), receivedMqttMessage ); - streamCapture.handleContent(); + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + streamCapture.handleContent(receivedMqttMessage); this.monitoringPage.update(); } @@ -439,7 +440,7 @@ public MonitoringPage() { // table to display topics topicsTable = new InformationTable( informationGroupTopics, - List.of( "Topic", "Number of received messages" ) + List.of( "Topic", "Number of received messages", "Recently received messages" ) ); im.registerInformation( topicsTable ); @@ -465,8 +466,16 @@ public void update() { topicsTable.addRow( "No topic subscriptions" ); } else { for ( String topic : topics.keySet() ) { - //TODO: format - topicsTable.addRow( topic, topics.get( topic ) ); + Long numberMessages = topics.get( topic ).get(); + if ( numberMessages != 0 ) { + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + List messages = streamCapture.getRecentMessages(namespace, topic); + topicsTable.addRow( topic, numberMessages, messages); + } else { + topicsTable.addRow( topic, topics.get( topic ), "No messages received yet." ); + } + + } } this.brokerInfo.setText( "Broker address: " + client.getConfig().getServerHost() diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 569de07803..7e6d696972 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -23,21 +23,14 @@ import org.bson.BsonString; import org.polypheny.db.PolyImplementation; import org.polypheny.db.adapter.DataStore; -import org.polypheny.db.algebra.AlgCollation; -import org.polypheny.db.algebra.AlgCollations; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; -import org.polypheny.db.algebra.core.Sort; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; -import org.polypheny.db.catalog.exceptions.GenericCatalogException; -import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; -import org.polypheny.db.catalog.exceptions.UnknownSchemaException; -import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.prepare.Context; @@ -45,8 +38,6 @@ import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; -import org.polypheny.db.transaction.TransactionManager; -import org.polypheny.db.util.Pair; import org.polypheny.db.util.PolyphenyHomeDirManager; @Slf4j @@ -57,15 +48,15 @@ public class StreamCapture { ReceivedMqttMessage receivedMqttMessage; - StreamCapture( final Transaction transaction, ReceivedMqttMessage receivedMqttMessage ) { + StreamCapture( final Transaction transaction ) { this.transaction = transaction; - this.receivedMqttMessage = receivedMqttMessage; + //this.receivedMqttMessage = receivedMqttMessage; } - public void handleContent() { - + public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { + this.receivedMqttMessage = receivedMqttMessage; if ( receivedMqttMessage.getStoreId() == 0 ) { //TODO: get store id of already existing or create new one. @@ -148,9 +139,17 @@ private long createNewCollection() { } + boolean saveContent() { + if ( this.receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { + insertDocument(); + } + return true; + } + + // added by Datomo public void insertDocument() { - String collectionName = "wohnzimmer." + this.receivedMqttMessage.getTopic(); + String collectionName = this.receivedMqttMessage.getNamespaceName() + "." + this.receivedMqttMessage.getTopic(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -167,20 +166,39 @@ public void insertDocument() { // we can then wrap the tree in an AlgRoot and execute it AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); // for inserts and all DML queries only a number is returned - String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); try { transaction.commit(); } catch ( TransactionException e ) { throw new RuntimeException( e ); } log.info( "inserted message as document" ); + } + + public List getRecentMessages( String namespaceName, String topic ) { + List listOfDocuments = scanCollection( namespaceName, topic ); + List listOfMessage = new ArrayList<>(); + //TODO: rmv debugging log: log.info( "listOfDocuments:{}", listOfDocuments ); + //only show last 20 Messages: + int indexLastMessage = 0; + if ( listOfDocuments.size() > 20 ) { + indexLastMessage = listOfDocuments.size() - 20; + } + for ( int i = indexLastMessage; i < listOfDocuments.size(); i++ ) { + String message = listOfDocuments.get( i ).split( "," )[1].trim(); + message = message.substring( message.indexOf( ':' ) ).trim(); + message = message.substring( message.indexOf( '"' ) + 1, message.lastIndexOf( '"' ) ); + listOfMessage.add( message ); + } + log.info( "extracted messages: {}", listOfMessage ); + return listOfMessage; } // added by Datomo - public void scanDocument() { - String collectionName = "users"; + public List scanCollection( String namespaceName, String topic ) { + String collectionName = namespaceName + "." + topic; Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -190,32 +208,25 @@ public void scanDocument() { // we can then wrap the tree in an AlgRoot and execute it AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); - String res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); - - } - - - boolean saveContent() { - if ( this.receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { - insertDocument(); + List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + List result = new ArrayList<>(); + for ( List objectsList : res ) { + result.add( objectsList.get( 0 ).toString() ); } - return true; + return result; } - String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { - //TODO: implement + List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { try { // Prepare PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); log.debug( "AlgRoot was prepared." ); - // todo transform into desired output format List> rows = result.getRows( statement, -1 ); - statement.getTransaction().commit(); - return rows.toString(); + return rows; } catch ( Throwable e ) { log.error( "Error during execution of stream capture query", e ); try { @@ -225,8 +236,6 @@ String executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final C } return null; } - } - } From 5088320b793f75d55a163d609a00ffe8a6c333c3 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 20 Jul 2023 13:17:46 +0200 Subject: [PATCH 054/114] added a reconnect button sometimes state of client shows Disconnected, therefore reconnect button --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 3730f4b83a..1984a0a41d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -212,7 +212,7 @@ public void shutdown() { } - private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { + public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { // TODO: Nachrichten an UI schicken falls Namespace name nicht geht long namespaceId = 0; @@ -408,9 +408,11 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; private final InformationGroup informationGroupMsg; + private final InformationGroup informationGroupReconn; private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; private final InformationAction msgButton; + private final InformationAction reconnButton; private final InformationText brokerInfo; private final InformationText clientInfo; private final InformationText namespaceName; @@ -457,6 +459,25 @@ public MonitoringPage() { } ).withParameters( "topic", "msg" ); im.registerInformation( msgButton ); + // Reconnection button + informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 4 ); + im.addGroup( informationGroupReconn ); + + reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { + String end = "Reconnected to broker"; + client.disconnect().whenComplete( ( disconn, throwable ) -> { + if ( throwable != null ) { + log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + } else { + run(); + update(); + } + } + ); + return end; + } ); + im.registerInformation( reconnButton ); + } From d777da609f6a669f3a08d38b41dc4c98bbb09539 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 20 Jul 2023 13:19:07 +0200 Subject: [PATCH 055/114] corrected code adding placements Errors, arise as mqtt query interface is not an adapter, which's id is needed to add a placement --- .../org/polypheny/db/mqtt/StreamCapture.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 7e6d696972..16cb7149c5 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -31,8 +31,11 @@ import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; +import org.polypheny.db.catalog.exceptions.UnknownAdapterException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; import org.polypheny.db.prepare.Context; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; @@ -88,7 +91,9 @@ long getCollectionId() { int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); if ( !collection.placements.contains( queryInterfaceId ) ) { log.info( "found matching collection!" ); - return collection.addPlacement( queryInterfaceId ).id; + //TODO: Nur AdapterID können als Placements hinzugefügt werden. nicht QueryInterfaceIds -> Marco fragen. + //catalog.addCollectionPlacement( queryInterfaceId, collection.id, PlacementType.MANUAL ); + return collection.id; } else { log.info( "found matching collection!" ); return collection.id; @@ -120,19 +125,22 @@ private long createNewCollection() { } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); return 0; - } catch ( TransactionException e ) { + } catch ( TransactionException e2 ) { log.error( "The commit after creating a new Collection could not be completed!" ); return 0; - } + } /*catch ( UnknownSchemaIdRuntimeException e3) { + MqttStreamServer. + //TODO: wenn man neue Namespace mit Collection erstellt nachdem man Mqtt interface erstellt hat, und diese für Mqtt benutzt -> muss man gerade Polypheny neu starten. + // saubere Lösung: methode getNamespaceId aufrufen, aber wie aufrufen aus StreamCapture? -> villeicht Button in Ui einfügen? + }*/ + //add placement - //TODO:insert Placements permanently: currently new placement is only inserted locally!!!! List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); - for ( int i = 0; i < collectionList.size(); i++ ) { - if ( collectionList.get( i ).name.equals( this.receivedMqttMessage.getTopic() ) ) { + for ( CatalogCollection collection : collectionList ) { + if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); - collectionList.set( i, collectionList.get( i ).addPlacement( queryInterfaceId ) ); - - return collectionList.get( i ).id; + catalog.addCollectionPlacement( queryInterfaceId, collection.id, PlacementType.MANUAL ); + return collection.id; } } return 0; From 52501f6f5aaaba933a76baaa8fe12f5430c0231a Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 20 Jul 2023 15:45:14 +0200 Subject: [PATCH 056/114] moved creating collections to MqttInterface --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 67 ++++++++++++++++++- .../org/polypheny/db/mqtt/StreamCapture.java | 42 ------------ 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 1984a0a41d..be4c3701cb 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -35,14 +35,19 @@ import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; +import org.polypheny.db.adapter.DataStore; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.PlacementType; +import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; import org.polypheny.db.catalog.exceptions.GenericCatalogException; import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.catalog.exceptions.UnknownUserException; +import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; @@ -55,6 +60,7 @@ import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; @@ -229,7 +235,6 @@ public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) namespaceId = schema.id; } else { log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); - log.info( "Please change the name or the type to {} to use the existing namespace.", schema.getNamespaceType() ); throw new RuntimeException(); } } else { @@ -283,6 +288,9 @@ protected void reloadSettings( List updatedSettings ) { this.namespaceId = namespaceId1; this.namespace = newNamespaceName; } + //create collections + //TODO: problem, if there is already collection with this name existing and has data inside: for getting the messages this can be problemeatic! + //TODO: mit marco abklären!!! break; case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); @@ -291,6 +299,7 @@ protected void reloadSettings( List updatedSettings ) { this.namespaceId = namespaceId2; this.namespaceType = newNamespaceType; } + //TODO: problem vrom above reggarding creation of new collections break; //TODO: handle change of Database maybe? } @@ -319,6 +328,9 @@ public void subscribe( String topic ) { log.info( "Subscription was not successfull. Please try again." ); } else { this.topics.put( topic, new AtomicLong(0)); + if ( !collectionExists( topic ) ) { + createNewCollection( topic ); + } log.info( "Successful subscription to topic:{}.", topic ); } } ); @@ -327,6 +339,59 @@ public void subscribe( String topic ) { } + private boolean collectionExists(String topic) { + Catalog catalog = Catalog.getInstance(); + List collectionList = catalog.getCollections( this.namespaceId, null ); + for ( CatalogCollection collection : collectionList ) { + if ( collection.name.equals( topic ) ) { + return true; + } + } + return false; + } + + + private void createNewCollection(String topic) { + Catalog catalog = Catalog.getInstance(); + Transaction transaction = getTransaction(); + //TODO: Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; + Statement statement = transaction.createStatement(); + + //need to create new Collection + try { + List dataStores = new ArrayList<>(); + DdlManager.getInstance().createCollection( + this.namespaceId, + topic, + true, //only creates collection if it does not already exist. + dataStores.size() == 0 ? null : dataStores, + PlacementType.MANUAL, + statement ); + log.info( "Created new collection with name: {}", topic ); + transaction.commit(); + } catch ( EntityAlreadyExistsException e ) { + log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); + return; + } catch ( TransactionException e2 ) { + log.error( "The commit after creating a new Collection could not be completed!" ); + return; + } /*catch ( UnknownSchemaIdRuntimeException e3 ) { + MqttStreamServer. + //TODO: wenn man neue Namespace mit Collection erstellt nachdem man Mqtt interface erstellt hat, und diese für Mqtt benutzt -> muss man gerade Polypheny neu starten. + // saubere Lösung: methode getNamespaceId aufrufen, aber wie aufrufen aus StreamCapture? -> villeicht Button in Ui einfügen? + }*/ + + //add placement + List collectionList2 = catalog.getCollections( this.namespaceId, null ); + for ( CatalogCollection collection : collectionList2 ) { + if ( collection.name.equals( topic ) ) { + // TODO: fix issue with placement! + //catalog.addCollectionPlacement( this.getQueryInterfaceId(), collection.id, PlacementType.MANUAL ); + } + } + } + + public void unsubscribe( List topics ) { for ( String t : topics ) { unsubscribe( t ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 16cb7149c5..46f85eadfe 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -100,49 +100,7 @@ long getCollectionId() { } } } - return createNewCollection(); - } - - - private long createNewCollection() { - Catalog catalog = Catalog.getInstance(); - - //Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; - Statement statement = transaction.createStatement(); - - try { - List dataStores = new ArrayList<>(); - DdlManager.getInstance().createCollection( - this.receivedMqttMessage.getNamespaceId(), - this.receivedMqttMessage.getTopic(), - true, //only creates collection if it does not already exist. - dataStores.size() == 0 ? null : dataStores, - PlacementType.MANUAL, - statement ); - log.info( "Created new collection with name: {}", this.receivedMqttMessage.getTopic() ); - transaction.commit(); - } catch ( EntityAlreadyExistsException e ) { - log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); - return 0; - } catch ( TransactionException e2 ) { - log.error( "The commit after creating a new Collection could not be completed!" ); - return 0; - } /*catch ( UnknownSchemaIdRuntimeException e3) { - MqttStreamServer. - //TODO: wenn man neue Namespace mit Collection erstellt nachdem man Mqtt interface erstellt hat, und diese für Mqtt benutzt -> muss man gerade Polypheny neu starten. - // saubere Lösung: methode getNamespaceId aufrufen, aber wie aufrufen aus StreamCapture? -> villeicht Button in Ui einfügen? - }*/ - - //add placement - List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); - for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); - catalog.addCollectionPlacement( queryInterfaceId, collection.id, PlacementType.MANUAL ); - return collection.id; - } - } return 0; } From 04d1fee4c2b4e495c0111cf09ccbb998987d9fb8 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 20 Jul 2023 19:42:30 +0200 Subject: [PATCH 057/114] implemented message count for each topic Now StreamCapture.getMessages returns list of MqttMessages with all messages of a collection. Umschalten von collection pro topic und Collection pro Interface ist bereits implementiert --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 77 +++++++++++++++---- .../org/polypheny/db/mqtt/StreamCapture.java | 31 +++----- 2 files changed, 74 insertions(+), 34 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index be4c3701cb..118418cb11 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -57,7 +56,6 @@ import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; import org.polypheny.db.information.InformationText; -import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; @@ -85,6 +83,8 @@ public void start() { // TODO namespace und type müssen dringend eingegeben werden, so machen wie bei topics? mqttDefaultSettings.put( "namespace", "public" ); mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); + mqttDefaultSettings.put( "collection per topic", "TRUE" ); + mqttDefaultSettings.put( "collection name", null ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -107,8 +107,10 @@ public static class MqttStreamServer extends QueryInterface { public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), - new QueryInterfaceSettingString( "namespace", false, true, true, null ), + new QueryInterfaceSettingString( "namespace", false, true, true, "public" ), new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "RELATIONAL", "DOCUMENT", "GRAPH" ) ) ), + new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), + new QueryInterfaceSettingString( "collection name", true, true, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ) ); @@ -126,6 +128,8 @@ public static class MqttStreamServer extends QueryInterface { private long namespaceId; private NamespaceType namespaceType; + private boolean collectionPerTopic; + private String collectionName; private long databaseId; @@ -157,7 +161,8 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.namespaceType = type; this.namespaceId = namespaceId; } - + this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); + this.collectionName = settings.get( "collection name" ); } @@ -292,6 +297,7 @@ protected void reloadSettings( List updatedSettings ) { //TODO: problem, if there is already collection with this name existing and has data inside: for getting the messages this can be problemeatic! //TODO: mit marco abklären!!! break; + case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); long namespaceId2 = this.getNamespaceId( this.namespace, newNamespaceType ); @@ -301,7 +307,16 @@ protected void reloadSettings( List updatedSettings ) { } //TODO: problem vrom above reggarding creation of new collections break; - //TODO: handle change of Database maybe? + + case "collection per topic": + this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + //TODO: if false: create collection with name + break; + case "collection name": + this.collectionName = this.getCurrentSettings().get( "collection name" ); + //TODO: create Collection + break; + } } } @@ -327,9 +342,29 @@ public void subscribe( String topic ) { if ( throwable != null ) { log.info( "Subscription was not successfull. Please try again." ); } else { - this.topics.put( topic, new AtomicLong(0)); + //create collection: + /* + if(collectionPerTopic){ + if ( !collectionExists( topic ) ) { + createNewCollection( topic ); + this.topics.put( topic, new AtomicLong(0)); + this.settings.put( topic, String.valueOf( 0 ) ); + } else {//new AtomicLong( getMessageCount( topic ) ) + this.topics.put( topic, new AtomicLong( Long.valueOf( this.settings.get( topic ) ) ) ); + } + + } else { + if ( !collectionExists( this.collectionName ) ) { + createNewCollection( this.collectionName ); + } + } + */ + if ( !collectionExists( topic ) ) { + this.topics.put( topic, new AtomicLong(0)); createNewCollection( topic ); + } else { + this.topics.put( topic, new AtomicLong(getMessageCount( topic ))); } log.info( "Successful subscription to topic:{}.", topic ); } @@ -339,6 +374,13 @@ public void subscribe( String topic ) { } + private long getMessageCount( String topic ) { + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + List messages = streamCapture.getMessages( namespace, topic ); + return messages.size(); + } + + private boolean collectionExists(String topic) { Catalog catalog = Catalog.getInstance(); List collectionList = catalog.getCollections( this.namespaceId, null ); @@ -406,7 +448,7 @@ public void unsubscribe( String topic ) { log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { this.topics.remove( topic ); - //TODO: remove collction in StreamCapture + log.info( "Unsubscribed from topic:{}.", topic ); } } ); @@ -551,14 +593,23 @@ public void update() { if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { + //TODO: distinguisch btw. Collecion per topic and per StreamInterface for ( String topic : topics.keySet() ) { - Long numberMessages = topics.get( topic ).get(); - if ( numberMessages != 0 ) { - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - List messages = streamCapture.getRecentMessages(namespace, topic); - topicsTable.addRow( topic, numberMessages, messages); + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + List messages = streamCapture.getMessages( namespace, topic ); + if ( messages.isEmpty() ) { + topicsTable.addRow( topic, 0, "No messages received yet." ); } else { - topicsTable.addRow( topic, topics.get( topic ), "No messages received yet." ); + //only show last 20 Messages: + int indexLastMessage = 0; + if ( messages.size() > 20 ) { + indexLastMessage = messages.size() - 20; + } + List recentMessages = new ArrayList<>(); + for ( int i = indexLastMessage; i < messages.size(); i++ ) { + recentMessages.add( messages.get( i ).getMessage() ); + } + topicsTable.addRow( topic, topics.get( topic ), recentMessages ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 46f85eadfe..1d629735a7 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -22,20 +22,13 @@ import org.bson.BsonDocument; import org.bson.BsonString; import org.polypheny.db.PolyImplementation; -import org.polypheny.db.adapter.DataStore; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; -import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; -import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; -import org.polypheny.db.catalog.exceptions.UnknownAdapterException; -import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; -import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.QueryInterfaceManager; -import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; import org.polypheny.db.prepare.Context; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; @@ -138,26 +131,22 @@ public void insertDocument() { } catch ( TransactionException e ) { throw new RuntimeException( e ); } - log.info( "inserted message as document" ); } - public List getRecentMessages( String namespaceName, String topic ) { - List listOfDocuments = scanCollection( namespaceName, topic ); - List listOfMessage = new ArrayList<>(); - //TODO: rmv debugging log: log.info( "listOfDocuments:{}", listOfDocuments ); - //only show last 20 Messages: - int indexLastMessage = 0; - if ( listOfDocuments.size() > 20 ) { - indexLastMessage = listOfDocuments.size() - 20; - } - for ( int i = indexLastMessage; i < listOfDocuments.size(); i++ ) { - String message = listOfDocuments.get( i ).split( "," )[1].trim(); + public List getMessages( String namespaceName, String collectionName ) { + List listOfDocuments = scanCollection( namespaceName, collectionName ); + List listOfMessage = new ArrayList<>(); + for ( String document : listOfDocuments ) { + String[] documentAsList = document.split( "," ); + String topic = documentAsList[0].substring( documentAsList[0].indexOf( ':' ) ); + topic = topic.substring( topic.indexOf( '"' ) + 1, topic.lastIndexOf( '"' ) ); + + String message = documentAsList[1].trim(); message = message.substring( message.indexOf( ':' ) ).trim(); message = message.substring( message.indexOf( '"' ) + 1, message.lastIndexOf( '"' ) ); - listOfMessage.add( message ); + listOfMessage.add( new MqttMessage( message, topic ) ); } - log.info( "extracted messages: {}", listOfMessage ); return listOfMessage; } From c017cac2ae359e75de150e6a0883be1567406835 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 21 Jul 2023 13:39:13 +0200 Subject: [PATCH 058/114] implemented a option to save all messages in one collection --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 150 +++++++++++------- .../db/mqtt/ReceivedMqttMessage.java | 7 +- .../org/polypheny/db/mqtt/StreamCapture.java | 19 +-- 3 files changed, 105 insertions(+), 71 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 118418cb11..a5e2b7721e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -80,7 +80,6 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - // TODO namespace und type müssen dringend eingegeben werden, so machen wie bei topics? mqttDefaultSettings.put( "namespace", "public" ); mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); mqttDefaultSettings.put( "collection per topic", "TRUE" ); @@ -142,8 +141,9 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); // Add information page this.monitoringPage = new MonitoringPage(); - this.broker = settings.get( "broker" ); - this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ) ); + this.broker = settings.get( "broker" ).trim(); + this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ).trim() ); + //TODO: currently: setting user and database with default value from Catalog for getTransaction method -> E schönerer Weg? // then current databaseid and userid are set. this.databaseId = Catalog.defaultDatabaseId; @@ -153,7 +153,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.userId = statement.getPrepareContext().getCurrentUserId(); this.databaseId = statement.getPrepareContext().getDatabaseId(); - String name = settings.get( "namespace" ); + String name = settings.get( "namespace" ).trim(); NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); long namespaceId = getNamespaceId( name, type ); if ( namespaceId != 0 ) { @@ -161,8 +161,16 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.namespaceType = type; this.namespaceId = namespaceId; } + this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); - this.collectionName = settings.get( "collection name" ); + this.collectionName = settings.get( "collection name" ).trim(); + if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { + if ( !this.collectionName.equals( "null" ) ){ + createNewCollection( this.collectionName ); + } else { + log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); + } + } } @@ -189,10 +197,6 @@ public void run() { log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - /**List topicsList = new ArrayList<>( List.of( this.settings.get( "topics" ).split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { - topicsList.set( i, topicsList.get(i).trim() ); - }**/ subscribe( topicsToList( this.settings.get( "topics" ) ) ); } } @@ -214,7 +218,6 @@ public void shutdown() { log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); } else { log.info( "{} stopped.", INTERFACE_NAME ); - //topics.clear(); monitoringPage.remove(); } } @@ -287,13 +290,15 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - String newNamespaceName = this.getCurrentSettings().get( "namespace" ); + String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); long namespaceId1 = this.getNamespaceId( newNamespaceName, this.namespaceType ); if ( namespaceId1 != 0 ) { this.namespaceId = namespaceId1; this.namespace = newNamespaceName; } - //create collections + //TODO: create collections + special case beachten!! + //TODO: abfrage ob sich auch namespaceType geändert hat: in dem Fall erst im namesoace type case collecitons erstellen! + //TODO: problem, if there is already collection with this name existing and has data inside: for getting the messages this can be problemeatic! //TODO: mit marco abklären!!! break; @@ -310,11 +315,23 @@ protected void reloadSettings( List updatedSettings ) { case "collection per topic": this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); - //TODO: if false: create collection with name + if ( !this.collectionPerTopic && !updatedSettings.contains( "collection name" ) && !collectionExists( this.collectionName ) ) { + if ( !this.collectionName.equals( "null" ) ){ + createNewCollection( this.collectionName ); + } else { + log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); + } + } break; case "collection name": - this.collectionName = this.getCurrentSettings().get( "collection name" ); - //TODO: create Collection + this.collectionName = this.getCurrentSettings().get( "collection name" ).trim(); + if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { + if ( !this.collectionName.equals( "null" ) ){ + createNewCollection( this.collectionName ); + } else { + log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); + } + } break; } @@ -342,30 +359,21 @@ public void subscribe( String topic ) { if ( throwable != null ) { log.info( "Subscription was not successfull. Please try again." ); } else { - //create collection: - /* - if(collectionPerTopic){ - if ( !collectionExists( topic ) ) { + if( collectionPerTopic ) { + if ( collectionExists( topic ) ) { + this.topics.put( topic, new AtomicLong( getMessages( topic ).size() ) ); + } else { + this.topics.put( topic, new AtomicLong( 0 ) ); createNewCollection( topic ); - this.topics.put( topic, new AtomicLong(0)); - this.settings.put( topic, String.valueOf( 0 ) ); - } else {//new AtomicLong( getMessageCount( topic ) ) - this.topics.put( topic, new AtomicLong( Long.valueOf( this.settings.get( topic ) ) ) ); } - } else { - if ( !collectionExists( this.collectionName ) ) { - createNewCollection( this.collectionName ); + long messageCount = getMessages( topic ).size(); + if ( messageCount == 0 ) { + this.topics.put( topic, new AtomicLong( 0 ) ); + } else { + this.topics.put( topic, new AtomicLong( messageCount ) ); } } - */ - - if ( !collectionExists( topic ) ) { - this.topics.put( topic, new AtomicLong(0)); - createNewCollection( topic ); - } else { - this.topics.put( topic, new AtomicLong(getMessageCount( topic ))); - } log.info( "Successful subscription to topic:{}.", topic ); } } ); @@ -374,18 +382,35 @@ public void subscribe( String topic ) { } - private long getMessageCount( String topic ) { + /** + * + * @param topic + * @return list of MqttMessages belonging to given topic. + */ + private List getMessages( String topic ) { StreamCapture streamCapture = new StreamCapture( getTransaction() ); - List messages = streamCapture.getMessages( namespace, topic ); - return messages.size(); + List messages; + if( this.collectionPerTopic ) { + messages = streamCapture.getMessages( namespace, topic ); + } else { + messages = streamCapture.getMessages( namespace, this.collectionName ); + //TODO: test if insert of Interface collection was implemented! + messages.removeIf( mqttMessage -> !Objects.equals( mqttMessage.getTopic(), topic ) ); + } + return messages; } - private boolean collectionExists(String topic) { + /** + * + * @param collectionName + * @return true: collection already exists, false: collection does not exist. + */ + private boolean collectionExists(String collectionName) { Catalog catalog = Catalog.getInstance(); List collectionList = catalog.getCollections( this.namespaceId, null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( topic ) ) { + if ( collection.name.equals( collectionName ) ) { return true; } } @@ -393,7 +418,7 @@ private boolean collectionExists(String topic) { } - private void createNewCollection(String topic) { + private void createNewCollection(String collectionName) { Catalog catalog = Catalog.getInstance(); Transaction transaction = getTransaction(); //TODO: Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; @@ -404,12 +429,12 @@ private void createNewCollection(String topic) { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( this.namespaceId, - topic, + collectionName, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created new collection with name: {}", topic ); + log.info( "Created new collection with name: {}", collectionName ); transaction.commit(); } catch ( EntityAlreadyExistsException e ) { log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); @@ -426,7 +451,7 @@ private void createNewCollection(String topic) { //add placement List collectionList2 = catalog.getCollections( this.namespaceId, null ); for ( CatalogCollection collection : collectionList2 ) { - if ( collection.name.equals( topic ) ) { + if ( collection.name.equals( collectionName ) ) { // TODO: fix issue with placement! //catalog.addCollectionPlacement( this.getQueryInterfaceId(), collection.id, PlacementType.MANUAL ); } @@ -444,11 +469,9 @@ public void unsubscribe( List topics ) { public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); } else { this.topics.remove( topic ); - log.info( "Unsubscribed from topic:{}.", topic ); } } ); @@ -458,8 +481,13 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - - ReceivedMqttMessage receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), subMsg.getTopic().toString() ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId ); + ReceivedMqttMessage receivedMqttMessage; + String topic = subMsg.getTopic().toString(); + if ( this.collectionPerTopic ) { + receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, topic ); + } else { + receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + } MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? @@ -572,15 +600,21 @@ public MonitoringPage() { reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { String end = "Reconnected to broker"; - client.disconnect().whenComplete( ( disconn, throwable ) -> { - if ( throwable != null ) { - log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); - } else { - run(); - update(); + //TOD0: abfrage nach state und jenach dem erst disconn oder nicht + if ( client.getState().toString().equals( "DISCONNECTED" ) ) { + run(); + update(); + } else { + client.disconnect().whenComplete( ( disconn, throwable ) -> { + if ( throwable != null ) { + log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + } else { + run(); + update(); + } } - } - ); + ); + } return end; } ); im.registerInformation( reconnButton ); @@ -593,10 +627,8 @@ public void update() { if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - //TODO: distinguisch btw. Collecion per topic and per StreamInterface for ( String topic : topics.keySet() ) { - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - List messages = streamCapture.getMessages( namespace, topic ); + List messages = getMessages( topic ); if ( messages.isEmpty() ) { topicsTable.addRow( topic, 0, "No messages received yet." ); } else { @@ -611,8 +643,6 @@ public void update() { } topicsTable.addRow( topic, topics.get( topic ), recentMessages ); } - - } } this.brokerInfo.setText( "Broker address: " + client.getConfig().getServerHost() diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java index 84eb2d97bf..0553e4aa17 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java @@ -38,9 +38,11 @@ public class ReceivedMqttMessage { @Getter @Setter private long storeId; // the ID of collection/graph/table... the place where info is/should be saved + @Getter + private final String collectionName; // if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is name of the topic + // if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is the name of the common collection - - public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, long storeId, String uniqueNameOfInterface, long databaseId, int userId ) { + public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, long storeId, String uniqueNameOfInterface, long databaseId, int userId, String collectionName ) { this.msg = msg; this.namespaceName = namespaceName; this.namespaceId = namespaceId; @@ -49,6 +51,7 @@ public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespac this.databaseId = databaseId; this.userId = userId; this.storeId = storeId; + this.collectionName = collectionName; } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 1d629735a7..f2fb6da2f0 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -80,7 +80,7 @@ long getCollectionId() { //check for collection with same name List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.receivedMqttMessage.getTopic() ) ) { + if ( collection.name.equals( this.receivedMqttMessage.getCollectionName() ) ) { int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); if ( !collection.placements.contains( queryInterfaceId ) ) { log.info( "found matching collection!" ); @@ -100,15 +100,15 @@ long getCollectionId() { boolean saveContent() { if ( this.receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { - insertDocument(); + insertDocument(this.receivedMqttMessage.getCollectionName()); } return true; } // added by Datomo - public void insertDocument() { - String collectionName = this.receivedMqttMessage.getNamespaceName() + "." + this.receivedMqttMessage.getTopic(); + public void insertDocument( String collectionName) { + String sqlCollectionName = this.receivedMqttMessage.getNamespaceName() + "." + collectionName; Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -116,11 +116,11 @@ public void insertDocument() { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); - //TODO: change to id: + //TODO: vll: data type einfügen document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); document.put( "content", new BsonString( this.receivedMqttMessage.getMessage() ) ); - AlgNode algNode = builder.docInsert( statement, collectionName, document ).build(); + AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); // we can then wrap the tree in an AlgRoot and execute it AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); @@ -135,6 +135,7 @@ public void insertDocument() { public List getMessages( String namespaceName, String collectionName ) { + //TODO: data type abfragen vll: List listOfDocuments = scanCollection( namespaceName, collectionName ); List listOfMessage = new ArrayList<>(); for ( String document : listOfDocuments ) { @@ -152,14 +153,14 @@ public List getMessages( String namespaceName, String collectionNam // added by Datomo - public List scanCollection( String namespaceName, String topic ) { - String collectionName = namespaceName + "." + topic; + public List scanCollection( String namespaceName, String collectionName ) { + String sqlCollectionName = namespaceName + "." + collectionName; Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed AlgBuilder builder = AlgBuilder.create( statement ); - AlgNode algNode = builder.docScan( statement, collectionName ).build(); + AlgNode algNode = builder.docScan( statement, sqlCollectionName ).build(); // we can then wrap the tree in an AlgRoot and execute it AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); From 599f4188d68c650570bc50378f62b9100312bd20 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 21 Jul 2023 14:26:40 +0200 Subject: [PATCH 059/114] imploved Monitoring Page --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index a5e2b7721e..255ec48c8d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -46,12 +46,14 @@ import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.catalog.exceptions.UnknownUserException; +import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.information.InformationAction; import org.polypheny.db.information.InformationGroup; +import org.polypheny.db.information.InformationKeyValue; import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; @@ -548,10 +550,6 @@ private class MonitoringPage { private final InformationTable topicsTable; private final InformationAction msgButton; private final InformationAction reconnButton; - private final InformationText brokerInfo; - private final InformationText clientInfo; - private final InformationText namespaceName; - public MonitoringPage() { @@ -562,12 +560,19 @@ public MonitoringPage() { informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); - brokerInfo = new InformationText( informationGroupInfo, "0.0.0.0:0000" ); - im.registerInformation( brokerInfo ); - clientInfo = new InformationText( informationGroupInfo, "o" ); - im.registerInformation( clientInfo ); - namespaceName = new InformationText( informationGroupInfo, "namespace" ); - im.registerInformation( namespaceName ); + InformationKeyValue brokerKv = new InformationKeyValue( informationGroupInfo ); + im.registerInformation( brokerKv ); + informationGroupInfo.setRefreshFunction( () -> { + brokerKv.putPair( "Broker address", client.getConfig().getServerHost() ); + brokerKv.putPair( "Broker port", client.getConfig().getServerPort() + "" ); + brokerKv.putPair( "Broker version of MQTT", client.getConfig().getMqttVersion() + "" ); + brokerKv.putPair( "Client state", client.getState() + "" ); + //TODO: get relevant info for cient identifier + brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier()+ "" ); + //TODO: check this after having SSL Configuration. + brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); + } ); + informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); @@ -628,31 +633,26 @@ public void update() { topicsTable.addRow( "No topic subscriptions" ); } else { for ( String topic : topics.keySet() ) { - List messages = getMessages( topic ); - if ( messages.isEmpty() ) { + List mqttMessageList = getMessages( topic ); + if ( mqttMessageList.isEmpty() ) { topicsTable.addRow( topic, 0, "No messages received yet." ); } else { //only show last 20 Messages: int indexLastMessage = 0; - if ( messages.size() > 20 ) { - indexLastMessage = messages.size() - 20; + if ( mqttMessageList.size() > 20 ) { + indexLastMessage = mqttMessageList.size() - 20; } List recentMessages = new ArrayList<>(); - for ( int i = indexLastMessage; i < messages.size(); i++ ) { - recentMessages.add( messages.get( i ).getMessage() ); + for ( int i = indexLastMessage; i < mqttMessageList.size(); i++ ) { + recentMessages.add( mqttMessageList.get( i ).getMessage() ); + } + topicsTable.addRow( topic, topics.get( topic ), "" ); + for ( String message : recentMessages ) { + topicsTable.addRow( "", "", message ); } - topicsTable.addRow( topic, topics.get( topic ), recentMessages ); } } } - this.brokerInfo.setText( "Broker address: " + client.getConfig().getServerHost() - + "\n" + "Broker port:" + client.getConfig().getServerPort() - + "\n Broker version of MQTT : " + String.valueOf( client.getConfig().getMqttVersion() ) - + "\n" + "SSL configuration: " + client.getConfig().getSslConfig() ); - //TODO: check this after having SSL Configuration. - this.clientInfo.setText( "Client state: " + String.valueOf( client.getState() ) ); - this.namespaceName.setText( "Namespace name: " + namespace ); - } From add043ae0d52ad4fc9152bc4786c7de45989a501 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 21 Jul 2023 16:23:36 +0200 Subject: [PATCH 060/114] small fixes --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 48 +++++++------------ .../org/polypheny/db/mqtt/StreamCapture.java | 10 +--- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 255ec48c8d..459c10331d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -45,6 +45,7 @@ import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; import org.polypheny.db.catalog.exceptions.UnknownDatabaseException; import org.polypheny.db.catalog.exceptions.UnknownSchemaException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; import org.polypheny.db.catalog.exceptions.UnknownUserException; import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.ddl.DdlManager; @@ -157,12 +158,9 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au String name = settings.get( "namespace" ).trim(); NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); - long namespaceId = getNamespaceId( name, type ); - if ( namespaceId != 0 ) { - this.namespace = name; - this.namespaceType = type; - this.namespaceId = namespaceId; - } + this.namespaceId = getNamespaceId( name, type ); + this.namespace = name; + this.namespaceType = type; this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); this.collectionName = settings.get( "collection name" ).trim(); @@ -230,7 +228,7 @@ public void shutdown() { public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - long namespaceId = 0; + long namespaceId; Catalog catalog = Catalog.getInstance(); if ( catalog.checkIfExistsSchema( this.databaseId, namespaceName ) ) { @@ -244,11 +242,10 @@ public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) if ( schema.namespaceType == namespaceType ) { namespaceId = schema.id; } else { - log.info( "There is already a namespace existing in this database with the given name but of type {}.", schema.getNamespaceType() ); - throw new RuntimeException(); + throw new RuntimeException("There is already a namespace existing in this database with the given name but of another type."); } } else { - + //create new namespace long id = catalog.addNamespace( namespaceName, this.databaseId, this.userId, namespaceType ); try { catalog.commit(); @@ -293,25 +290,19 @@ protected void reloadSettings( List updatedSettings ) { case "namespace": String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - long namespaceId1 = this.getNamespaceId( newNamespaceName, this.namespaceType ); - if ( namespaceId1 != 0 ) { - this.namespaceId = namespaceId1; - this.namespace = newNamespaceName; - } + this.namespaceId = this.getNamespaceId( newNamespaceName, this.namespaceType ); + this.namespace = newNamespaceName; + //TODO: create collections + special case beachten!! //TODO: abfrage ob sich auch namespaceType geändert hat: in dem Fall erst im namesoace type case collecitons erstellen! - //TODO: problem, if there is already collection with this name existing and has data inside: for getting the messages this can be problemeatic! - //TODO: mit marco abklären!!! break; case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - long namespaceId2 = this.getNamespaceId( this.namespace, newNamespaceType ); - if ( namespaceId2 != 0 ) { - this.namespaceId = namespaceId2; - this.namespaceType = newNamespaceType; - } + this.namespaceId = this.getNamespaceId( this.namespace, newNamespaceType ); + this.namespaceType = newNamespaceType; + //TODO: problem vrom above reggarding creation of new collections break; @@ -396,7 +387,6 @@ private List getMessages( String topic ) { messages = streamCapture.getMessages( namespace, topic ); } else { messages = streamCapture.getMessages( namespace, this.collectionName ); - //TODO: test if insert of Interface collection was implemented! messages.removeIf( mqttMessage -> !Objects.equals( mqttMessage.getTopic(), topic ) ); } return messages; @@ -423,7 +413,6 @@ private boolean collectionExists(String collectionName) { private void createNewCollection(String collectionName) { Catalog catalog = Catalog.getInstance(); Transaction transaction = getTransaction(); - //TODO: Catalog.PlacementType placementType = Catalog.PlacementType.AUTOMATIC; Statement statement = transaction.createStatement(); //need to create new Collection @@ -444,11 +433,9 @@ private void createNewCollection(String collectionName) { } catch ( TransactionException e2 ) { log.error( "The commit after creating a new Collection could not be completed!" ); return; - } /*catch ( UnknownSchemaIdRuntimeException e3 ) { - MqttStreamServer. - //TODO: wenn man neue Namespace mit Collection erstellt nachdem man Mqtt interface erstellt hat, und diese für Mqtt benutzt -> muss man gerade Polypheny neu starten. - // saubere Lösung: methode getNamespaceId aufrufen, aber wie aufrufen aus StreamCapture? -> villeicht Button in Ui einfügen? - }*/ + } catch ( UnknownSchemaIdRuntimeException e3 ) { + this.namespaceId = getNamespaceId( this.namespace, this.namespaceType ); + } //add placement List collectionList2 = catalog.getCollections( this.namespaceId, null ); @@ -567,8 +554,7 @@ public MonitoringPage() { brokerKv.putPair( "Broker port", client.getConfig().getServerPort() + "" ); brokerKv.putPair( "Broker version of MQTT", client.getConfig().getMqttVersion() + "" ); brokerKv.putPair( "Client state", client.getState() + "" ); - //TODO: get relevant info for cient identifier - brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier()+ "" ); + brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get()+ "" ); //TODO: check this after having SSL Configuration. brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); } ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index f2fb6da2f0..abefd4cdf8 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -54,9 +54,7 @@ public class StreamCapture { public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { this.receivedMqttMessage = receivedMqttMessage; if ( receivedMqttMessage.getStoreId() == 0 ) { - //TODO: get store id of already existing or create new one. - - //TODO: maybe do this with interface + //TODO: maybe do this with interface for different types if ( receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { long newstoreId = getCollectionId(); if ( newstoreId != 0 ) { @@ -65,10 +63,6 @@ public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { } } boolean saved = saveContent(); - //TODO: gescheite Tests -// Catalog catalog = Catalog.getInstance(); -// CatalogSchema schema = null; -// schema = catalog.getSchema( stream.getNamespaceID() ); } @@ -116,7 +110,6 @@ public void insertDocument( String collectionName) { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); - //TODO: vll: data type einfügen document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); document.put( "content", new BsonString( this.receivedMqttMessage.getMessage() ) ); @@ -135,7 +128,6 @@ public void insertDocument( String collectionName) { public List getMessages( String namespaceName, String collectionName ) { - //TODO: data type abfragen vll: List listOfDocuments = scanCollection( namespaceName, collectionName ); List listOfMessage = new ArrayList<>(); for ( String document : listOfDocuments ) { From 7671e8766179e398faa989fc260fba8ede2edebd Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 27 Jul 2023 15:14:30 +0200 Subject: [PATCH 061/114] minor fixes and reformatted file + changed monitoring settings --- .../polypheny/db/config/RuntimeConfig.java | 8 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 136 ++++++++---------- 2 files changed, 64 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java index bd85e9f135..9b8d03955f 100644 --- a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java +++ b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java @@ -159,21 +159,21 @@ public enum RuntimeConfig { DYNAMIC_QUERYING( "statistics/useDynamicQuerying", "Use statistics for query assistance.", - true, + false, ConfigType.BOOLEAN, "statisticSettingsGroup" ), STATISTICS_ON_STARTUP( "statistics/statisticsOnStartup", "Whether to build statistics for all stored data on system startup.", - true, + false, ConfigType.BOOLEAN, "statisticSettingsGroup" ), ACTIVE_TRACKING( "statistics/activeTracking", "All transactions are tracked and statistics collected during execution.", - true, + false, ConfigType.BOOLEAN, "statisticSettingsGroup" ), @@ -418,7 +418,7 @@ public enum RuntimeConfig { MONITORING_QUEUE_ACTIVE( "runtime/monitoringQueueActive", "Enables automatic monitoring of executed events in workload monitoring. If disabled no events are captured, hence the queue remains empty. This also effects routing!", - true, + false, ConfigType.BOOLEAN, "monitoringSettingsQueueGroup" ), diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 459c10331d..eee269966c 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -47,7 +47,6 @@ import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; import org.polypheny.db.catalog.exceptions.UnknownUserException; -import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.iface.Authenticator; import org.polypheny.db.iface.QueryInterface; @@ -58,7 +57,9 @@ import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; -import org.polypheny.db.information.InformationText; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.rex.RexBuilder; +import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; @@ -84,9 +85,9 @@ public void start() { mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); mqttDefaultSettings.put( "namespace", "public" ); - mqttDefaultSettings.put( "namespace type", "RELATIONAL" ); + mqttDefaultSettings.put( "namespace type", "DOCUMENT" ); mqttDefaultSettings.put( "collection per topic", "TRUE" ); - mqttDefaultSettings.put( "collection name", null ); + mqttDefaultSettings.put( "collection name", "default" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -109,8 +110,10 @@ public static class MqttStreamServer extends QueryInterface { public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), + //TODO: namespace modifieable machen new QueryInterfaceSettingString( "namespace", false, true, true, "public" ), - new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "RELATIONAL", "DOCUMENT", "GRAPH" ) ) ), + // "RELATIONAL", "GRAPH" type are not supported. + new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT") ) ), new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "collection name", true, true, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ) @@ -120,11 +123,8 @@ public static class MqttStreamServer extends QueryInterface { private final String broker; @Getter private final int brokerPort; - private Map topics = new ConcurrentHashMap<>(); - private Mqtt3AsyncClient client; - private String namespace; private long namespaceId; @@ -132,11 +132,8 @@ public static class MqttStreamServer extends QueryInterface { private NamespaceType namespaceType; private boolean collectionPerTopic; private String collectionName; - - private long databaseId; - - private int userId; - + private final long databaseId; + private final int userId; private final MonitoringPage monitoringPage; @@ -147,14 +144,8 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.broker = settings.get( "broker" ).trim(); this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ).trim() ); - //TODO: currently: setting user and database with default value from Catalog for getTransaction method -> E schönerer Weg? - // then current databaseid and userid are set. this.databaseId = Catalog.defaultDatabaseId; this.userId = Catalog.defaultUserId; - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - this.userId = statement.getPrepareContext().getCurrentUserId(); - this.databaseId = statement.getPrepareContext().getDatabaseId(); String name = settings.get( "namespace" ).trim(); NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); @@ -165,7 +156,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); this.collectionName = settings.get( "collection name" ).trim(); if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ){ + if ( !this.collectionName.equals( "null" ) ) { createNewCollection( this.collectionName ); } else { log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); @@ -228,7 +219,7 @@ public void shutdown() { public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - long namespaceId; + long namespaceId = 0; Catalog catalog = Catalog.getInstance(); if ( catalog.checkIfExistsSchema( this.databaseId, namespaceName ) ) { @@ -242,7 +233,7 @@ public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) if ( schema.namespaceType == namespaceType ) { namespaceId = schema.id; } else { - throw new RuntimeException("There is already a namespace existing in this database with the given name but of another type."); + log.info("There is already a namespace existing in this database with the given name but of another type."); } } else { //create new namespace @@ -294,8 +285,8 @@ protected void reloadSettings( List updatedSettings ) { this.namespace = newNamespaceName; //TODO: create collections + special case beachten!! + //TODO: schauen wie updatedSettings aussieht, ist es die Reihenfolge von Setiigns oder willkürlich? //TODO: abfrage ob sich auch namespaceType geändert hat: in dem Fall erst im namesoace type case collecitons erstellen! - break; case "namespace type": @@ -308,8 +299,9 @@ protected void reloadSettings( List updatedSettings ) { case "collection per topic": this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + //TODO: Reihenfolge von updatedSettings anschauen! if ( !this.collectionPerTopic && !updatedSettings.contains( "collection name" ) && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ){ + if ( !this.collectionName.equals( "null" ) ) { createNewCollection( this.collectionName ); } else { log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); @@ -319,7 +311,7 @@ protected void reloadSettings( List updatedSettings ) { case "collection name": this.collectionName = this.getCurrentSettings().get( "collection name" ).trim(); if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ){ + if ( !this.collectionName.equals( "null" ) ) { createNewCollection( this.collectionName ); } else { log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); @@ -342,30 +334,18 @@ void subscribe( List newTopics ) { /** * subscribes to one given topic and adds it to the List topics. * - * @param topic the topic the client should subscribe to. + * @param topic: the topic the client should subscribe to. */ public void subscribe( String topic ) { client.subscribeWith().topicFilter( topic ).callback( subMsg -> { - log.info( "Received message from topic {}.", subMsg.getTopic() ); processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { - log.info( "Subscription was not successfull. Please try again." ); + throw new RuntimeException( "Subscription was not successful. Please try again." ); } else { - if( collectionPerTopic ) { - if ( collectionExists( topic ) ) { - this.topics.put( topic, new AtomicLong( getMessages( topic ).size() ) ); - } else { - this.topics.put( topic, new AtomicLong( 0 ) ); - createNewCollection( topic ); - } - } else { - long messageCount = getMessages( topic ).size(); - if ( messageCount == 0 ) { - this.topics.put( topic, new AtomicLong( 0 ) ); - } else { - this.topics.put( topic, new AtomicLong( messageCount ) ); - } + this.topics.put( topic, new AtomicLong( 0 ) ); + if ( collectionPerTopic && !collectionExists( topic ) ) { + createNewCollection( topic ); } log.info( "Successful subscription to topic:{}.", topic ); } @@ -376,14 +356,13 @@ public void subscribe( String topic ) { /** - * * @param topic * @return list of MqttMessages belonging to given topic. */ private List getMessages( String topic ) { StreamCapture streamCapture = new StreamCapture( getTransaction() ); List messages; - if( this.collectionPerTopic ) { + if ( this.collectionPerTopic ) { messages = streamCapture.getMessages( namespace, topic ); } else { messages = streamCapture.getMessages( namespace, this.collectionName ); @@ -394,11 +373,10 @@ private List getMessages( String topic ) { /** - * * @param collectionName * @return true: collection already exists, false: collection does not exist. */ - private boolean collectionExists(String collectionName) { + private boolean collectionExists( String collectionName ) { Catalog catalog = Catalog.getInstance(); List collectionList = catalog.getCollections( this.namespaceId, null ); for ( CatalogCollection collection : collectionList ) { @@ -410,9 +388,9 @@ private boolean collectionExists(String collectionName) { } - private void createNewCollection(String collectionName) { + private void createNewCollection( String collectionName ) { Catalog catalog = Catalog.getInstance(); - Transaction transaction = getTransaction(); + Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); //need to create new Collection @@ -434,7 +412,7 @@ private void createNewCollection(String collectionName) { log.error( "The commit after creating a new Collection could not be completed!" ); return; } catch ( UnknownSchemaIdRuntimeException e3 ) { - this.namespaceId = getNamespaceId( this.namespace, this.namespaceType ); + this.namespaceId = getNamespaceId( this.namespace, this.namespaceType ); } //add placement @@ -472,21 +450,24 @@ void processMsg( Mqtt3Publish subMsg ) { Statement statement = transaction.createStatement(); ReceivedMqttMessage receivedMqttMessage; String topic = subMsg.getTopic().toString(); + + topics.replace( topic, new AtomicLong( topics.get( topic ).incrementAndGet() ) ); + if ( this.collectionPerTopic ) { receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, topic ); } else { receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? if ( !Objects.equals( content, "" ) ) { - Long incrementedCount = topics.get( receivedMqttMessage.getTopic() ).incrementAndGet(); - topics.replace( receivedMqttMessage.getTopic(), new AtomicLong( incrementedCount ) ); + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + streamCapture.handleContent( receivedMqttMessage ); + this.monitoringPage.update(); } - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - streamCapture.handleContent(receivedMqttMessage); - this.monitoringPage.update(); + } @@ -513,6 +494,15 @@ private Transaction getTransaction() { } + private AlgBuilder getAlgBuilder() { + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + final RexBuilder rexBuilder = new RexBuilder( statement.getTransaction().getTypeFactory() ); + final AlgOptCluster cluster = AlgOptCluster.create( statement.getQueryProcessor().getPlanner(), rexBuilder ); + return AlgBuilder.create( statement, cluster ); + } + + @Override public void languageChange() { @@ -531,10 +521,11 @@ private class MonitoringPage { private final InformationGroup informationGroupTopics; - private final InformationGroup informationGroupMsg; + private final InformationGroup informationGroupPub; private final InformationGroup informationGroupReconn; private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; + private final InformationKeyValue brokerKv; private final InformationAction msgButton; private final InformationAction reconnButton; @@ -546,39 +537,25 @@ public MonitoringPage() { informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); - - InformationKeyValue brokerKv = new InformationKeyValue( informationGroupInfo ); + brokerKv = new InformationKeyValue( informationGroupInfo ); im.registerInformation( brokerKv ); - informationGroupInfo.setRefreshFunction( () -> { - brokerKv.putPair( "Broker address", client.getConfig().getServerHost() ); - brokerKv.putPair( "Broker port", client.getConfig().getServerPort() + "" ); - brokerKv.putPair( "Broker version of MQTT", client.getConfig().getMqttVersion() + "" ); - brokerKv.putPair( "Client state", client.getState() + "" ); - brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get()+ "" ); - //TODO: check this after having SSL Configuration. - brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); - } ); - + informationGroupInfo.setRefreshFunction( this::update ); informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); - im.addPage( informationPage ); im.addGroup( informationGroupTopics ); - - // table to display topics topicsTable = new InformationTable( informationGroupTopics, List.of( "Topic", "Number of received messages", "Recently received messages" ) ); - im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); //TODO: rmv button - informationGroupMsg = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); - im.addGroup( informationGroupMsg ); + informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); + im.addGroup( informationGroupPub ); - msgButton = new InformationAction( informationGroupMsg, "Send a msg", ( parameters ) -> { + msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); return end; @@ -586,9 +563,8 @@ public MonitoringPage() { im.registerInformation( msgButton ); // Reconnection button - informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 4 ); + informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); im.addGroup( informationGroupReconn ); - reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { String end = "Reconnected to broker"; //TOD0: abfrage nach state und jenach dem erst disconn oder nicht @@ -639,6 +615,14 @@ public void update() { } } } + + brokerKv.putPair( "Broker address", client.getConfig().getServerHost() ); + brokerKv.putPair( "Broker port", client.getConfig().getServerPort() + "" ); + brokerKv.putPair( "Broker version of MQTT", client.getConfig().getMqttVersion() + "" ); + brokerKv.putPair( "Client state", client.getState() + "" ); + brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get() + "" ); + //TODO: check this after having SSL Configuration. + brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); } From db2757f877dd597d70fbfc47b40e995c6aaa0700 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 27 Jul 2023 17:07:03 +0200 Subject: [PATCH 062/114] Minor fixes - changed logs to exception - using blocking client for reconnecting --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 66 +++++++++---------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index eee269966c..b8ed49c92e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -84,7 +84,7 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - mqttDefaultSettings.put( "namespace", "public" ); + mqttDefaultSettings.put( "namespace", "namespace1" ); mqttDefaultSettings.put( "namespace type", "DOCUMENT" ); mqttDefaultSettings.put( "collection per topic", "TRUE" ); mqttDefaultSettings.put( "collection name", "default" ); @@ -111,11 +111,11 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), //TODO: namespace modifieable machen - new QueryInterfaceSettingString( "namespace", false, true, true, "public" ), + new QueryInterfaceSettingString( "namespace", false, true, true, "namespace1" ), // "RELATIONAL", "GRAPH" type are not supported. new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT") ) ), new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), - new QueryInterfaceSettingString( "collection name", true, true, true, null ), + new QueryInterfaceSettingString( "collection name", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ) ); @@ -155,11 +155,11 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); this.collectionName = settings.get( "collection name" ).trim(); - if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ) { + if ( !this.collectionPerTopic ) { + if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { + throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); + } else if ( !collectionExists( this.collectionName ) ) { createNewCollection( this.collectionName ); - } else { - log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); } } } @@ -185,7 +185,7 @@ public void run() { .send() .whenComplete( ( connAck, throwable ) -> { if ( throwable != null ) { - log.error( "Connection to broker could not be established. Please delete and recreate the Plug-In." ); + throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); subscribe( topicsToList( this.settings.get( "topics" ) ) ); @@ -206,7 +206,7 @@ public void shutdown() { client.disconnect().whenComplete( ( disconn, throwable ) -> { if ( throwable != null ) { - log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker +":" + brokerPort + ". Please try again.",throwable); } else { log.info( "{} stopped.", INTERFACE_NAME ); monitoringPage.remove(); @@ -233,7 +233,7 @@ public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) if ( schema.namespaceType == namespaceType ) { namespaceId = schema.id; } else { - log.info("There is already a namespace existing in this database with the given name but of another type."); + throw new RuntimeException("There is already a namespace existing in this database with the given name but of another type."); } } else { //create new namespace @@ -301,20 +301,20 @@ protected void reloadSettings( List updatedSettings ) { this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); //TODO: Reihenfolge von updatedSettings anschauen! if ( !this.collectionPerTopic && !updatedSettings.contains( "collection name" ) && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ) { + if ( ! (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { createNewCollection( this.collectionName ); } else { - log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); + throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); } } break; case "collection name": this.collectionName = this.getCurrentSettings().get( "collection name" ).trim(); if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { - if ( !this.collectionName.equals( "null" ) ) { + if ( ! (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { createNewCollection( this.collectionName ); } else { - log.error( "Collection per topic is set as FALSE but no collection name was given! Please enter a collection name." ); + throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); } } break; @@ -341,12 +341,13 @@ public void subscribe( String topic ) { processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { - throw new RuntimeException( "Subscription was not successful. Please try again." ); + throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topics.put( topic, new AtomicLong( 0 ) ); if ( collectionPerTopic && !collectionExists( topic ) ) { createNewCollection( topic ); } + //TODO: rmv, only for debugging needed log.info( "Successful subscription to topic:{}.", topic ); } } ); @@ -403,25 +404,13 @@ private void createNewCollection( String collectionName ) { dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); - log.info( "Created new collection with name: {}", collectionName ); transaction.commit(); - } catch ( EntityAlreadyExistsException e ) { - log.error( "The generation of the collection was not possible because there is a collection already existing with this name." ); - return; - } catch ( TransactionException e2 ) { - log.error( "The commit after creating a new Collection could not be completed!" ); - return; + } catch ( EntityAlreadyExistsException | TransactionException e ) { + throw new RuntimeException( "Error while creating a new collection:", e ); } catch ( UnknownSchemaIdRuntimeException e3 ) { + //TODO: überlegen, was in diesem Fall passiert. this.namespaceId = getNamespaceId( this.namespace, this.namespaceType ); - } - - //add placement - List collectionList2 = catalog.getCollections( this.namespaceId, null ); - for ( CatalogCollection collection : collectionList2 ) { - if ( collection.name.equals( collectionName ) ) { - // TODO: fix issue with placement! - //catalog.addCollectionPlacement( this.getQueryInterfaceId(), collection.id, PlacementType.MANUAL ); - } + throw new RuntimeException( "Collection could not be created, but the correct namespace id was determined." ); } } @@ -436,10 +425,9 @@ public void unsubscribe( List topics ) { public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - log.error( String.format( "Topic %s could not be unsubscribed.", topic ) ); + throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); } else { this.topics.remove( topic ); - log.info( "Unsubscribed from topic:{}.", topic ); } } ); } @@ -451,8 +439,10 @@ void processMsg( Mqtt3Publish subMsg ) { ReceivedMqttMessage receivedMqttMessage; String topic = subMsg.getTopic().toString(); - topics.replace( topic, new AtomicLong( topics.get( topic ).incrementAndGet() ) ); + Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); + topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); + //TODO: storeiD beachten was ist das? if ( this.collectionPerTopic ) { receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, topic ); } else { @@ -567,20 +557,24 @@ public MonitoringPage() { im.addGroup( informationGroupReconn ); reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { String end = "Reconnected to broker"; - //TOD0: abfrage nach state und jenach dem erst disconn oder nicht if ( client.getState().toString().equals( "DISCONNECTED" ) ) { run(); update(); } else { + client.toBlocking().disconnect(); + run(); + update(); + /* client.disconnect().whenComplete( ( disconn, throwable ) -> { if ( throwable != null ) { - log.info( "{} could not disconnect from MQTT broker {}:{}. Please try again.", INTERFACE_NAME, broker, brokerPort ); + throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ".", throwable ); } else { run(); update(); } } ); + */ } return end; } ); From b0fe5d0294ee35ff183a5c285cffb3183e202842 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 27 Jul 2023 19:29:01 +0200 Subject: [PATCH 063/114] made both classes independent of ids --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 100 ++++++++++-------- .../db/mqtt/ReceivedMqttMessage.java | 10 +- .../org/polypheny/db/mqtt/StreamCapture.java | 29 ++--- 3 files changed, 66 insertions(+), 73 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index b8ed49c92e..255569438f 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -37,6 +37,7 @@ import org.polypheny.db.adapter.DataStore; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.Pattern; import org.polypheny.db.catalog.Catalog.PlacementType; import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.catalog.entity.CatalogSchema; @@ -126,9 +127,6 @@ public static class MqttStreamServer extends QueryInterface { private Map topics = new ConcurrentHashMap<>(); private Mqtt3AsyncClient client; private String namespace; - - private long namespaceId; - private NamespaceType namespaceType; private boolean collectionPerTopic; private String collectionName; @@ -149,7 +147,9 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au String name = settings.get( "namespace" ).trim(); NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); - this.namespaceId = getNamespaceId( name, type ); + if ( !namespaceExists( name, type ) ) { + createNamespace( name, type ); + } this.namespace = name; this.namespaceType = type; @@ -217,38 +217,58 @@ public void shutdown() { } - public long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { - // TODO: Nachrichten an UI schicken falls Namespace name nicht geht - long namespaceId = 0; + private boolean namespaceExists (String namespaceName, NamespaceType namespaceType) { Catalog catalog = Catalog.getInstance(); - if ( catalog.checkIfExistsSchema( this.databaseId, namespaceName ) ) { - CatalogSchema schema = null; - try { - schema = catalog.getSchema( this.databaseId, namespaceName ); - } catch ( UnknownSchemaException e ) { - throw new RuntimeException( e ); - } - assert schema != null; - if ( schema.namespaceType == namespaceType ) { - namespaceId = schema.id; - } else { - throw new RuntimeException("There is already a namespace existing in this database with the given name but of another type."); - } + if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + getExistingNamespaceId( namespaceName, namespaceType ); + return true; } else { - //create new namespace - long id = catalog.addNamespace( namespaceName, this.databaseId, this.userId, namespaceType ); - try { - catalog.commit(); - namespaceId = id; - } catch ( NoTablePrimaryKeyException e ) { - throw new RuntimeException( e ); - } + return false; + } + } + + + private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) { + Catalog catalog = Catalog.getInstance(); + if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { + return getExistingNamespaceId( namespaceName, namespaceType ); + } else { + return createNamespace( namespaceName, namespaceType ); } - return namespaceId; } + + private long getExistingNamespaceId(String namespaceName, NamespaceType namespaceType) { + Catalog catalog = Catalog.getInstance(); + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + throw new RuntimeException( e ); + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + return schema.id; + } else { + throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type." ); + } + } + + + private long createNamespace(String namespaceName, NamespaceType namespaceType) { + Catalog catalog = Catalog.getInstance(); + long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); + try { + catalog.commit(); + return id; + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException( e ); + } + } + + @Override protected void reloadSettings( List updatedSettings ) { for ( String changedSetting : updatedSettings ) { @@ -281,7 +301,6 @@ protected void reloadSettings( List updatedSettings ) { case "namespace": String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - this.namespaceId = this.getNamespaceId( newNamespaceName, this.namespaceType ); this.namespace = newNamespaceName; //TODO: create collections + special case beachten!! @@ -291,7 +310,6 @@ protected void reloadSettings( List updatedSettings ) { case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - this.namespaceId = this.getNamespaceId( this.namespace, newNamespaceType ); this.namespaceType = newNamespaceType; //TODO: problem vrom above reggarding creation of new collections @@ -379,13 +397,9 @@ private List getMessages( String topic ) { */ private boolean collectionExists( String collectionName ) { Catalog catalog = Catalog.getInstance(); - List collectionList = catalog.getCollections( this.namespaceId, null ); - for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( collectionName ) ) { - return true; - } - } - return false; + Pattern pattern = new Pattern( collectionName ); + List collectionList = catalog.getCollections( getNamespaceId( this.namespace, this.namespaceType ), pattern ); + return !collectionList.isEmpty(); } @@ -398,7 +412,7 @@ private void createNewCollection( String collectionName ) { try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( - this.namespaceId, + getNamespaceId( this.namespace, this.namespaceType ), collectionName, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, @@ -408,9 +422,7 @@ private void createNewCollection( String collectionName ) { } catch ( EntityAlreadyExistsException | TransactionException e ) { throw new RuntimeException( "Error while creating a new collection:", e ); } catch ( UnknownSchemaIdRuntimeException e3 ) { - //TODO: überlegen, was in diesem Fall passiert. - this.namespaceId = getNamespaceId( this.namespace, this.namespaceType ); - throw new RuntimeException( "Collection could not be created, but the correct namespace id was determined." ); + throw new RuntimeException( "New collection could not be created.", e3 ); } } @@ -444,9 +456,9 @@ void processMsg( Mqtt3Publish subMsg ) { //TODO: storeiD beachten was ist das? if ( this.collectionPerTopic ) { - receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, topic ); + receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); } else { - receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, this.namespaceId, this.namespaceType, 0, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java index 0553e4aa17..1cd8ea6452 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java @@ -36,21 +36,17 @@ public class ReceivedMqttMessage { @Getter private final int userId; @Getter - @Setter - private long storeId; // the ID of collection/graph/table... the place where info is/should be saved - @Getter private final String collectionName; // if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is name of the topic - // if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is the name of the common collection + // if MqttStreamServer.collectionPerTopic = FALSE, then collectionName is the name of the common collection - public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, long storeId, String uniqueNameOfInterface, long databaseId, int userId, String collectionName ) { + public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, String uniqueNameOfInterface, long databaseId, int userId, String collectionName ) { this.msg = msg; this.namespaceName = namespaceName; - this.namespaceId = namespaceId; this.namespaceType = namespaceType; + this.namespaceId = namespaceId; this.uniqueNameOfInterface = uniqueNameOfInterface; this.databaseId = databaseId; this.userId = userId; - this.storeId = storeId; this.collectionName = collectionName; } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index abefd4cdf8..fc193fcf0b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -27,8 +27,11 @@ import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.Pattern; import org.polypheny.db.catalog.entity.CatalogCollection; -import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.prepare.Context; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; @@ -46,30 +49,19 @@ public class StreamCapture { StreamCapture( final Transaction transaction ) { this.transaction = transaction; - //this.receivedMqttMessage = receivedMqttMessage; - } public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { this.receivedMqttMessage = receivedMqttMessage; - if ( receivedMqttMessage.getStoreId() == 0 ) { - //TODO: maybe do this with interface for different types - if ( receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { - long newstoreId = getCollectionId(); - if ( newstoreId != 0 ) { - receivedMqttMessage.setStoreId( newstoreId ); - } - } - } - boolean saved = saveContent(); + insertDocument( this.receivedMqttMessage.getCollectionName() ); } /** * @return the id of the collection that was either already existing with the topic as name or that was newly created */ - long getCollectionId() { +/* long getCollectionId() { Catalog catalog = Catalog.getInstance(); //check for collection with same name List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); @@ -90,14 +82,7 @@ long getCollectionId() { return 0; } - - - boolean saveContent() { - if ( this.receivedMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { - insertDocument(this.receivedMqttMessage.getCollectionName()); - } - return true; - } + */ // added by Datomo From 40a6cd7a64cd20662a9d385a0cdbcf1409c556a9 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 28 Jul 2023 19:06:28 +0200 Subject: [PATCH 064/114] debugging --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 150 ++++++++++++++---- 1 file changed, 120 insertions(+), 30 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 255569438f..c918464861 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -111,10 +111,9 @@ public static class MqttStreamServer extends QueryInterface { public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), - //TODO: namespace modifieable machen new QueryInterfaceSettingString( "namespace", false, true, true, "namespace1" ), - // "RELATIONAL", "GRAPH" type are not supported. - new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT") ) ), + // "RELATIONAL", "GRAPH" type are not supported yet. + new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH") ) ), new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "collection name", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ) @@ -157,7 +156,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.collectionName = settings.get( "collection name" ).trim(); if ( !this.collectionPerTopic ) { if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { - throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); + throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { createNewCollection( this.collectionName ); } @@ -252,7 +251,7 @@ private long getExistingNamespaceId(String namespaceName, NamespaceType namespac if ( schema.namespaceType == namespaceType ) { return schema.id; } else { - throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type." ); + throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type. Please change the namespace name or the type." ); } } @@ -300,40 +299,119 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - this.namespace = newNamespaceName; + /* + try { + this.wait(2000); + } catch ( InterruptedException e ) { + throw new RuntimeException( e ); + } + this.notify(); + + */ +//TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! + /* + try { + client.wait(); + } catch ( InterruptedException e ) { + throw new RuntimeException( e ); + } - //TODO: create collections + special case beachten!! - //TODO: schauen wie updatedSettings aussieht, ist es die Reihenfolge von Setiigns oder willkürlich? - //TODO: abfrage ob sich auch namespaceType geändert hat: in dem Fall erst im namesoace type case collecitons erstellen! + + */ + + synchronized ( this.namespace ) { + String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); + if ( updatedSettings.contains( "namespace type" ) ) { + if ( updatedSettings.indexOf( "namespace type" ) < updatedSettings.indexOf( "namespace" ) ) { + NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + try { + if ( !namespaceExists( newNamespaceName, type ) ) { + createNamespace( newNamespaceName, type ); + } + this.namespace = newNamespaceName; + this.namespaceType = type; + } catch ( RuntimeException e ) { + this.settings.put( "namespace", this.namespace ); + this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + throw new RuntimeException( e ); + } + } // else checking for namespace happens in case "namespace type" + } else { + try { + if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { + createNamespace( newNamespaceName, this.namespaceType ); + } + this.namespace = newNamespaceName; + } catch ( RuntimeException e ) { + this.settings.put( "namespace", this.namespace ); + throw new RuntimeException( e ); + } + } + + //TODO: rmv sout: + System.out.println("namespace changed" + this.namespace); + } + + //client.notify(); break; case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - this.namespaceType = newNamespaceType; - - //TODO: problem vrom above reggarding creation of new collections + synchronized ( this.namespaceType ) { + if ( updatedSettings.contains( "namespace" ) ) { + if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespace type" ) ) { + String newName = this.getCurrentSettings().get( "namespace" ); + try { + if ( !namespaceExists( newName, newNamespaceType ) ) { + createNamespace( newName, newNamespaceType ); + } + this.namespace = newName; + this.namespaceType = newNamespaceType; + } catch ( RuntimeException e ) { + this.settings.put( "namespace", this.namespace ); + this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + throw new RuntimeException( e ); + } + } // else checking for namespace happens in case "namespace" + } else { + try { + if ( !namespaceExists( this.namespace, newNamespaceType ) ) { + createNamespace( this.namespace, newNamespaceType ); + } + this.namespaceType = newNamespaceType; + } catch ( RuntimeException e ) { + this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + throw new RuntimeException( e ); + } + } + } break; case "collection per topic": this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); - //TODO: Reihenfolge von updatedSettings anschauen! - if ( !this.collectionPerTopic && !updatedSettings.contains( "collection name" ) && !collectionExists( this.collectionName ) ) { - if ( ! (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { - createNewCollection( this.collectionName ); - } else { - throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); - } - } break; + case "collection name": - this.collectionName = this.getCurrentSettings().get( "collection name" ).trim(); - if ( !this.collectionPerTopic && !collectionExists( this.collectionName ) ) { - if ( ! (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { - createNewCollection( this.collectionName ); + String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); + boolean mode; + if (updatedSettings.contains( "collection per topic" ) ) { + mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + } else { + mode = this.collectionPerTopic; + } + if ( !mode ) { + if ( ! (newCollectionName.equals( "null" ) | newCollectionName.equals( "" ) ) ) { + if ( !collectionExists( newCollectionName ) ) { + createNewCollection( this.collectionName ); + } + this.collectionName = newCollectionName; } else { - throw new NullPointerException( "Collection per topic is set to FALSE but no collection name was given! Please enter a collection name." ); + this.settings.put( "collection name", this.collectionName ); + throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } + + } else { + this.collectionName = newCollectionName; } break; @@ -359,6 +437,16 @@ public void subscribe( String topic ) { processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { + // change settings: + List topicsList = topicsToList( this.settings.get( "topics" ) ); + StringBuilder stringBuilder = new StringBuilder(); + for ( String t : topicsList ) { + if ( !t.equals( topic ) ) { + stringBuilder.append( t ).append( "," ); + } + } + String topicsString = stringBuilder.toString(); + this.settings.replace( "topics", topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ) ); throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topics.put( topic, new AtomicLong( 0 ) ); @@ -407,8 +495,6 @@ private void createNewCollection( String collectionName ) { Catalog catalog = Catalog.getInstance(); Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - - //need to create new Collection try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( @@ -437,6 +523,8 @@ public void unsubscribe( List topics ) { public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { + // change settings: + this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); } else { this.topics.remove( topic ); @@ -454,7 +542,6 @@ void processMsg( Mqtt3Publish subMsg ) { Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); - //TODO: storeiD beachten was ist das? if ( this.collectionPerTopic ) { receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); } else { @@ -481,7 +568,10 @@ private static String extractPayload( Mqtt3Publish subMsg ) { public List topicsToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { - topicsList.set( i, topicsList.get( i ).trim() ); + String topic = topicsList.get( i ).trim(); + if ( !topic.isEmpty() ) { + topicsList.set( i, topic); + } } return topicsList; } From f3cda565d54743abcb82dc8cfacb47965f3a499d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 28 Jul 2023 19:06:45 +0200 Subject: [PATCH 065/114] inserted field source --- .../src/main/java/org/polypheny/db/mqtt/StreamCapture.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index fc193fcf0b..30dbaefd21 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -95,6 +95,7 @@ public void insertDocument( String collectionName) { // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); + document.put( "source", new BsonString( this.receivedMqttMessage.getUniqueNameOfInterface() ) ); document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); document.put( "content", new BsonString( this.receivedMqttMessage.getMessage() ) ); @@ -117,10 +118,10 @@ public List getMessages( String namespaceName, String collectionNam List listOfMessage = new ArrayList<>(); for ( String document : listOfDocuments ) { String[] documentAsList = document.split( "," ); - String topic = documentAsList[0].substring( documentAsList[0].indexOf( ':' ) ); + String topic = documentAsList[1].substring( documentAsList[0].indexOf( ':' ) ); topic = topic.substring( topic.indexOf( '"' ) + 1, topic.lastIndexOf( '"' ) ); - String message = documentAsList[1].trim(); + String message = documentAsList[2].trim(); message = message.substring( message.indexOf( ':' ) ).trim(); message = message.substring( message.indexOf( '"' ) + 1, message.lastIndexOf( '"' ) ); listOfMessage.add( new MqttMessage( message, topic ) ); From 97a9d9f0738813a49243f2be339cf57331b12281 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 29 Jul 2023 11:12:16 +0200 Subject: [PATCH 066/114] added function to create all necessary collections in reload method --- .../polypheny/db/stream/StreamMessage.java | 3 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 100 +++++++++++------- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java index 26e9995f0c..fc6fa0f6b2 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java @@ -16,7 +16,8 @@ package org.polypheny.db.stream; -public interface StreamMessage { +public interface StreamMessage { + T getMessage(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index c918464861..866fe95f0d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -113,7 +113,7 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), new QueryInterfaceSettingString( "namespace", false, true, true, "namespace1" ), // "RELATIONAL", "GRAPH" type are not supported yet. - new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH") ) ), + new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "collection name", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ) @@ -155,7 +155,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); this.collectionName = settings.get( "collection name" ).trim(); if ( !this.collectionPerTopic ) { - if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" ) ) ) { + if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { createNewCollection( this.collectionName ); @@ -205,7 +205,7 @@ public void shutdown() { client.disconnect().whenComplete( ( disconn, throwable ) -> { if ( throwable != null ) { - throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker +":" + brokerPort + ". Please try again.",throwable); + throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ". Please try again.", throwable ); } else { log.info( "{} stopped.", INTERFACE_NAME ); monitoringPage.remove(); @@ -216,8 +216,7 @@ public void shutdown() { } - - private boolean namespaceExists (String namespaceName, NamespaceType namespaceType) { + private boolean namespaceExists( String namespaceName, NamespaceType namespaceType ) { Catalog catalog = Catalog.getInstance(); if ( catalog.checkIfExistsSchema( Catalog.defaultDatabaseId, namespaceName ) ) { getExistingNamespaceId( namespaceName, namespaceType ); @@ -238,34 +237,33 @@ private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) } - - private long getExistingNamespaceId(String namespaceName, NamespaceType namespaceType) { - Catalog catalog = Catalog.getInstance(); - CatalogSchema schema = null; - try { - schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); - } catch ( UnknownSchemaException e ) { - throw new RuntimeException( e ); - } - assert schema != null; - if ( schema.namespaceType == namespaceType ) { - return schema.id; - } else { - throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type. Please change the namespace name or the type." ); + private long getExistingNamespaceId( String namespaceName, NamespaceType namespaceType ) { + Catalog catalog = Catalog.getInstance(); + CatalogSchema schema = null; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + throw new RuntimeException( e ); + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + return schema.id; + } else { + throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type. Please change the namespace name or the type." ); + } } - } - private long createNamespace(String namespaceName, NamespaceType namespaceType) { - Catalog catalog = Catalog.getInstance(); - long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); - try { - catalog.commit(); - return id; - } catch ( NoTablePrimaryKeyException e ) { - throw new RuntimeException( e ); + private long createNamespace( String namespaceName, NamespaceType namespaceType ) { + Catalog catalog = Catalog.getInstance(); + long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); + try { + catalog.commit(); + return id; + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException( e ); + } } - } @Override @@ -330,6 +328,7 @@ protected void reloadSettings( List updatedSettings ) { } this.namespace = newNamespaceName; this.namespaceType = type; + createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespace ); this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); @@ -342,6 +341,7 @@ protected void reloadSettings( List updatedSettings ) { createNamespace( newNamespaceName, this.namespaceType ); } this.namespace = newNamespaceName; + createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespace ); throw new RuntimeException( e ); @@ -349,7 +349,7 @@ protected void reloadSettings( List updatedSettings ) { } //TODO: rmv sout: - System.out.println("namespace changed" + this.namespace); + System.out.println( "namespace changed " + this.namespace ); } //client.notify(); @@ -367,6 +367,7 @@ protected void reloadSettings( List updatedSettings ) { } this.namespace = newName; this.namespaceType = newNamespaceType; + createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespace ); this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); @@ -379,6 +380,7 @@ protected void reloadSettings( List updatedSettings ) { createNamespace( this.namespace, newNamespaceType ); } this.namespaceType = newNamespaceType; + createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); @@ -389,22 +391,24 @@ protected void reloadSettings( List updatedSettings ) { case "collection per topic": this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + createAllCollections(); break; case "collection name": String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); boolean mode; - if (updatedSettings.contains( "collection per topic" ) ) { + if ( updatedSettings.contains( "collection per topic" ) ) { mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); } else { mode = this.collectionPerTopic; } if ( !mode ) { - if ( ! (newCollectionName.equals( "null" ) | newCollectionName.equals( "" ) ) ) { + if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { if ( !collectionExists( newCollectionName ) ) { createNewCollection( this.collectionName ); } this.collectionName = newCollectionName; + createAllCollections(); } else { this.settings.put( "collection name", this.collectionName ); throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); @@ -413,6 +417,7 @@ protected void reloadSettings( List updatedSettings ) { } else { this.collectionName = newCollectionName; } + break; } @@ -438,7 +443,7 @@ public void subscribe( String topic ) { } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { // change settings: - List topicsList = topicsToList( this.settings.get( "topics" ) ); + List topicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); StringBuilder stringBuilder = new StringBuilder(); for ( String t : topicsList ) { if ( !t.equals( topic ) ) { @@ -446,7 +451,11 @@ public void subscribe( String topic ) { } } String topicsString = stringBuilder.toString(); - this.settings.replace( "topics", topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ) ); + if ( !topicsString.isEmpty() ) { + topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); + } + this.settings.put( "topics", topicsString ); + log.info( "not successful: {}", topic ); throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topics.put( topic, new AtomicLong( 0 ) ); @@ -513,6 +522,25 @@ private void createNewCollection( String collectionName ) { } + private void createAllCollections() { + if ( this.collectionPerTopic ) { + for ( String t : this.topics.keySet() ) { + if ( !collectionExists( t ) ) { + createNewCollection( t ); + } + } + } else { + if ( !(this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { + if ( !collectionExists( this.collectionName ) ) { + createNewCollection( this.collectionName ); + } + } else { + throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + } + } + } + + public void unsubscribe( List topics ) { for ( String t : topics ) { unsubscribe( t ); @@ -524,7 +552,7 @@ public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { // change settings: - this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); + this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); } else { this.topics.remove( topic ); @@ -570,7 +598,7 @@ public List topicsToList( String topics ) { for ( int i = 0; i < topicsList.size(); i++ ) { String topic = topicsList.get( i ).trim(); if ( !topic.isEmpty() ) { - topicsList.set( i, topic); + topicsList.set( i, topic ); } } return topicsList; From a88e58447c2972d1a375cbb7aec377f3b4ac642d Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 29 Jul 2023 14:13:07 +0200 Subject: [PATCH 067/114] implemented queue for messages --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 185 +++++++++--------- 1 file changed, 91 insertions(+), 94 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 866fe95f0d..a9d11db024 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -26,8 +26,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -124,6 +126,7 @@ public static class MqttStreamServer extends QueryInterface { @Getter private final int brokerPort; private Map topics = new ConcurrentHashMap<>(); + private ConcurrentLinkedQueue messageQueue = new ConcurrentLinkedQueue<>(); private Mqtt3AsyncClient client; private String namespace; private NamespaceType namespaceType; @@ -471,20 +474,67 @@ public void subscribe( String topic ) { } - /** - * @param topic - * @return list of MqttMessages belonging to given topic. - */ - private List getMessages( String topic ) { - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - List messages; + public void unsubscribe( List topics ) { + for ( String t : topics ) { + unsubscribe( t ); + } + } + + + public void unsubscribe( String topic ) { + client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { + if ( throwable != null ) { + // change settings: + this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); + throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); + } else { + this.topics.remove( topic ); + } + } ); + } + + + void processMsg( Mqtt3Publish subMsg ) { + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + ReceivedMqttMessage receivedMqttMessage; + String topic = subMsg.getTopic().toString(); + String message = extractPayload( subMsg ); + MqttMessage mqttMessage = new MqttMessage( message, topic ); + Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); + topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); + addMessageToQueue( topic, message ); + if ( this.collectionPerTopic ) { - messages = streamCapture.getMessages( namespace, topic ); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); } else { - messages = streamCapture.getMessages( namespace, this.collectionName ); - messages.removeIf( mqttMessage -> !Objects.equals( mqttMessage.getTopic(), topic ) ); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } - return messages; + + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); + String content = streamProcessor.processStream(); + //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? + if ( !Objects.equals( content, "" ) ) { + StreamCapture streamCapture = new StreamCapture( getTransaction() ); + streamCapture.handleContent( receivedMqttMessage ); + } + } + + + private static String extractPayload( Mqtt3Publish subMsg ) { + return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); + } + + + public List topicsToList( String topics ) { + List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); + for ( int i = 0; i < topicsList.size(); i++ ) { + String topic = topicsList.get( i ).trim(); + if ( !topic.isEmpty() ) { + topicsList.set( i, topic ); + } + } + return topicsList; } @@ -541,67 +591,13 @@ private void createAllCollections() { } - public void unsubscribe( List topics ) { - for ( String t : topics ) { - unsubscribe( t ); - } - } - - - public void unsubscribe( String topic ) { - client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { - if ( throwable != null ) { - // change settings: - this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); - throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); - } else { - this.topics.remove( topic ); - } - } ); - } - - - void processMsg( Mqtt3Publish subMsg ) { - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - ReceivedMqttMessage receivedMqttMessage; - String topic = subMsg.getTopic().toString(); - - Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); - topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); - - if ( this.collectionPerTopic ) { - receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); + private void addMessageToQueue( String topic, String message ) { + if ( this.messageQueue.size() >= 20 ) { + this.messageQueue.poll(); + this.messageQueue.add( new String[]{ topic, message } ); } else { - receivedMqttMessage = new ReceivedMqttMessage( new MqttMessage( extractPayload( subMsg ), topic ), this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); - } - - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); - String content = streamProcessor.processStream(); - //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? - if ( !Objects.equals( content, "" ) ) { - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - streamCapture.handleContent( receivedMqttMessage ); - this.monitoringPage.update(); + this.messageQueue.add( new String[]{ topic, message } ); } - - } - - - private static String extractPayload( Mqtt3Publish subMsg ) { - return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); - } - - - public List topicsToList( String topics ) { - List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { - String topic = topicsList.get( i ).trim(); - if ( !topic.isEmpty() ) { - topicsList.set( i, topic ); - } - } - return topicsList; } @@ -640,11 +636,12 @@ private class MonitoringPage { private final InformationPage informationPage; private final InformationGroup informationGroupTopics; - + private final InformationGroup informationGroupMessage; private final InformationGroup informationGroupPub; private final InformationGroup informationGroupReconn; private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; + private final InformationTable messageTable; private final InformationKeyValue brokerKv; private final InformationAction msgButton; private final InformationAction reconnButton; @@ -653,7 +650,8 @@ private class MonitoringPage { public MonitoringPage() { InformationManager im = InformationManager.getInstance(); - informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).fullWidth().setLabel( "Interfaces" ); + informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).setLabel( "Interfaces" ); + im.addPage( informationPage ); informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); @@ -662,19 +660,26 @@ public MonitoringPage() { informationGroupInfo.setRefreshFunction( this::update ); informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); - im.addPage( informationPage ); im.addGroup( informationGroupTopics ); topicsTable = new InformationTable( informationGroupTopics, - List.of( "Topic", "Number of received messages", "Recently received messages" ) + List.of( "Topic", "Number of received messages" ) ); im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); + informationGroupMessage = new InformationGroup( informationPage, "Recently arrived messages" ).setOrder( 2 ); + im.addGroup( informationGroupMessage ); + messageTable = new InformationTable( + informationGroupMessage, + List.of( "Topic", "Message" ) + ); + im.registerInformation( messageTable ); + informationGroupMessage.setRefreshFunction( this::update ); + //TODO: rmv button informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupPub ); - msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); @@ -718,25 +723,17 @@ public void update() { if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - for ( String topic : topics.keySet() ) { - List mqttMessageList = getMessages( topic ); - if ( mqttMessageList.isEmpty() ) { - topicsTable.addRow( topic, 0, "No messages received yet." ); - } else { - //only show last 20 Messages: - int indexLastMessage = 0; - if ( mqttMessageList.size() > 20 ) { - indexLastMessage = mqttMessageList.size() - 20; - } - List recentMessages = new ArrayList<>(); - for ( int i = indexLastMessage; i < mqttMessageList.size(); i++ ) { - recentMessages.add( mqttMessageList.get( i ).getMessage() ); - } - topicsTable.addRow( topic, topics.get( topic ), "" ); - for ( String message : recentMessages ) { - topicsTable.addRow( "", "", message ); - } - } + for ( Entry t : topics.entrySet() ) { + topicsTable.addRow( t.getKey(), t.getValue() ); + } + } + + messageTable.reset(); + if ( messageQueue.isEmpty() ) { + messageTable.addRow( "No messages received yet." ); + } else { + for ( String[] message : messageQueue ) { + messageTable.addRow( List.of( message ) ); } } From 8138aca1621214b9131287b4d79909790379359a Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 29 Jul 2023 18:33:07 +0200 Subject: [PATCH 068/114] resolving concurrency changed collectionPerTopic type to AtomicBoolean --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index a9d11db024..d7cd583701 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -30,6 +30,8 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -130,7 +132,7 @@ public static class MqttStreamServer extends QueryInterface { private Mqtt3AsyncClient client; private String namespace; private NamespaceType namespaceType; - private boolean collectionPerTopic; + private AtomicBoolean collectionPerTopic; private String collectionName; private final long databaseId; private final int userId; @@ -155,9 +157,10 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.namespace = name; this.namespaceType = type; - this.collectionPerTopic = Boolean.parseBoolean( settings.get( "collection per topic" ) ); + + this.collectionPerTopic = new AtomicBoolean(Boolean.parseBoolean( settings.get( "collection per topic" ) ) ); this.collectionName = settings.get( "collection name" ).trim(); - if ( !this.collectionPerTopic ) { + if ( !this.collectionPerTopic.get() ) { if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { @@ -242,7 +245,7 @@ private long getNamespaceId( String namespaceName, NamespaceType namespaceType ) private long getExistingNamespaceId( String namespaceName, NamespaceType namespaceType ) { Catalog catalog = Catalog.getInstance(); - CatalogSchema schema = null; + CatalogSchema schema; try { schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); } catch ( UnknownSchemaException e ) { @@ -275,25 +278,21 @@ protected void reloadSettings( List updatedSettings ) { switch ( changedSetting ) { case "topics": List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); - List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { if ( !topics.containsKey( newTopic ) ) { topicsToSub.add( newTopic ); } } - if ( !topicsToSub.isEmpty() ) { subscribe( topicsToSub ); } - List topicsToUnsub = new ArrayList<>(); for ( String oldTopic : topics.keySet() ) { if ( !newTopicsList.contains( oldTopic ) ) { topicsToUnsub.add( oldTopic ); } } - if ( !topicsToUnsub.isEmpty() ) { unsubscribe( topicsToUnsub ); } @@ -319,7 +318,6 @@ protected void reloadSettings( List updatedSettings ) { */ - synchronized ( this.namespace ) { String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); if ( updatedSettings.contains( "namespace type" ) ) { @@ -350,13 +348,8 @@ protected void reloadSettings( List updatedSettings ) { throw new RuntimeException( e ); } } - - //TODO: rmv sout: - System.out.println( "namespace changed " + this.namespace ); } - - //client.notify(); - break; + break; case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); @@ -389,21 +382,21 @@ protected void reloadSettings( List updatedSettings ) { throw new RuntimeException( e ); } } + } - break; + break; case "collection per topic": - this.collectionPerTopic = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ) ); createAllCollections(); break; - case "collection name": String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); boolean mode; if ( updatedSettings.contains( "collection per topic" ) ) { mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); } else { - mode = this.collectionPerTopic; + mode = this.collectionPerTopic.get(); } if ( !mode ) { if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { @@ -420,9 +413,7 @@ protected void reloadSettings( List updatedSettings ) { } else { this.collectionName = newCollectionName; } - break; - } } } @@ -462,7 +453,7 @@ public void subscribe( String topic ) { throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topics.put( topic, new AtomicLong( 0 ) ); - if ( collectionPerTopic && !collectionExists( topic ) ) { + if ( collectionPerTopic.get() && !collectionExists( topic ) ) { createNewCollection( topic ); } //TODO: rmv, only for debugging needed @@ -504,13 +495,11 @@ void processMsg( Mqtt3Publish subMsg ) { Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); addMessageToQueue( topic, message ); - - if ( this.collectionPerTopic ) { + if ( this.collectionPerTopic.get() ) { receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); } else { receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? @@ -545,13 +534,13 @@ public List topicsToList( String topics ) { private boolean collectionExists( String collectionName ) { Catalog catalog = Catalog.getInstance(); Pattern pattern = new Pattern( collectionName ); - List collectionList = catalog.getCollections( getNamespaceId( this.namespace, this.namespaceType ), pattern ); + List collectionList = null; + collectionList = catalog.getCollections( getNamespaceId( this.namespace, this.namespaceType), pattern ); return !collectionList.isEmpty(); } private void createNewCollection( String collectionName ) { - Catalog catalog = Catalog.getInstance(); Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); try { @@ -573,7 +562,7 @@ private void createNewCollection( String collectionName ) { private void createAllCollections() { - if ( this.collectionPerTopic ) { + if ( this.collectionPerTopic.get() ) { for ( String t : this.topics.keySet() ) { if ( !collectionExists( t ) ) { createNewCollection( t ); @@ -636,14 +625,9 @@ private class MonitoringPage { private final InformationPage informationPage; private final InformationGroup informationGroupTopics; - private final InformationGroup informationGroupMessage; - private final InformationGroup informationGroupPub; - private final InformationGroup informationGroupReconn; - private final InformationGroup informationGroupInfo; private final InformationTable topicsTable; private final InformationTable messageTable; private final InformationKeyValue brokerKv; - private final InformationAction msgButton; private final InformationAction reconnButton; @@ -653,7 +637,7 @@ public MonitoringPage() { informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).setLabel( "Interfaces" ); im.addPage( informationPage ); - informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); + InformationGroup informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); brokerKv = new InformationKeyValue( informationGroupInfo ); im.registerInformation( brokerKv ); @@ -668,7 +652,7 @@ public MonitoringPage() { im.registerInformation( topicsTable ); informationGroupTopics.setRefreshFunction( this::update ); - informationGroupMessage = new InformationGroup( informationPage, "Recently arrived messages" ).setOrder( 2 ); + InformationGroup informationGroupMessage = new InformationGroup( informationPage, "Recently arrived messages" ).setOrder( 2 ); im.addGroup( informationGroupMessage ); messageTable = new InformationTable( informationGroupMessage, @@ -678,9 +662,9 @@ public MonitoringPage() { informationGroupMessage.setRefreshFunction( this::update ); //TODO: rmv button - informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); + InformationGroup informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupPub ); - msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { + InformationAction msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); return end; @@ -688,7 +672,7 @@ public MonitoringPage() { im.registerInformation( msgButton ); // Reconnection button - informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); + InformationGroup informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); im.addGroup( informationGroupReconn ); reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { String end = "Reconnected to broker"; @@ -719,6 +703,20 @@ public MonitoringPage() { public void update() { + /* TODO : rmv concurency trial + String s = namespace; + String c; + for ( ;; ) { + if ( s.equals( namespace ) ) { + c = "!"; + } else { + c = "."; + } + System.out.print(c); + } + + */ + topicsTable.reset(); if ( topics.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); From 4fdeeb292b7487474fd8c0c380a3d308b52a5cba Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 30 Jul 2023 12:23:35 +0200 Subject: [PATCH 069/114] Minor fix changed variable names, inserted TODOs where synchronized is needed. --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 109 ++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index d7cd583701..90d7e87e56 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -127,15 +127,16 @@ public static class MqttStreamServer extends QueryInterface { private final String broker; @Getter private final int brokerPort; - private Map topics = new ConcurrentHashMap<>(); + private Map topicsMap = new ConcurrentHashMap<>(); private ConcurrentLinkedQueue messageQueue = new ConcurrentLinkedQueue<>(); private Mqtt3AsyncClient client; - private String namespace; + private String namespaceName; private NamespaceType namespaceType; private AtomicBoolean collectionPerTopic; private String collectionName; private final long databaseId; private final int userId; + private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -154,7 +155,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au if ( !namespaceExists( name, type ) ) { createNamespace( name, type ); } - this.namespace = name; + this.namespaceName = name; this.namespaceType = type; @@ -274,13 +275,14 @@ private long createNamespace( String namespaceName, NamespaceType namespaceType @Override protected void reloadSettings( List updatedSettings ) { + //TODO: synch lock araound for schleife: for ( String changedSetting : updatedSettings ) { switch ( changedSetting ) { case "topics": List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { - if ( !topics.containsKey( newTopic ) ) { + if ( !topicsMap.containsKey( newTopic ) ) { topicsToSub.add( newTopic ); } } @@ -288,7 +290,7 @@ protected void reloadSettings( List updatedSettings ) { subscribe( topicsToSub ); } List topicsToUnsub = new ArrayList<>(); - for ( String oldTopic : topics.keySet() ) { + for ( String oldTopic : topicsMap.keySet() ) { if ( !newTopicsList.contains( oldTopic ) ) { topicsToUnsub.add( oldTopic ); } @@ -318,7 +320,7 @@ protected void reloadSettings( List updatedSettings ) { */ - synchronized ( this.namespace ) { + synchronized ( this.namespaceName ) { String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); if ( updatedSettings.contains( "namespace type" ) ) { if ( updatedSettings.indexOf( "namespace type" ) < updatedSettings.indexOf( "namespace" ) ) { @@ -327,11 +329,11 @@ protected void reloadSettings( List updatedSettings ) { if ( !namespaceExists( newNamespaceName, type ) ) { createNamespace( newNamespaceName, type ); } - this.namespace = newNamespaceName; + this.namespaceName = newNamespaceName; this.namespaceType = type; createAllCollections(); } catch ( RuntimeException e ) { - this.settings.put( "namespace", this.namespace ); + this.settings.put( "namespace", this.namespaceName ); this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } @@ -341,10 +343,10 @@ protected void reloadSettings( List updatedSettings ) { if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { createNamespace( newNamespaceName, this.namespaceType ); } - this.namespace = newNamespaceName; + this.namespaceName = newNamespaceName; createAllCollections(); } catch ( RuntimeException e ) { - this.settings.put( "namespace", this.namespace ); + this.settings.put( "namespace", this.namespaceName ); throw new RuntimeException( e ); } } @@ -353,7 +355,7 @@ protected void reloadSettings( List updatedSettings ) { case "namespace type": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - synchronized ( this.namespaceType ) { + //synchronized ( this.namespaceType ) { if ( updatedSettings.contains( "namespace" ) ) { if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespace type" ) ) { String newName = this.getCurrentSettings().get( "namespace" ); @@ -361,19 +363,19 @@ protected void reloadSettings( List updatedSettings ) { if ( !namespaceExists( newName, newNamespaceType ) ) { createNamespace( newName, newNamespaceType ); } - this.namespace = newName; + this.namespaceName = newName; this.namespaceType = newNamespaceType; createAllCollections(); } catch ( RuntimeException e ) { - this.settings.put( "namespace", this.namespace ); + this.settings.put( "namespace", this.namespaceName ); this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } } // else checking for namespace happens in case "namespace" } else { try { - if ( !namespaceExists( this.namespace, newNamespaceType ) ) { - createNamespace( this.namespace, newNamespaceType ); + if ( !namespaceExists( this.namespaceName, newNamespaceType ) ) { + createNamespace( this.namespaceName, newNamespaceType ); } this.namespaceType = newNamespaceType; createAllCollections(); @@ -383,7 +385,7 @@ protected void reloadSettings( List updatedSettings ) { } } - } + // } break; case "collection per topic": @@ -436,7 +438,7 @@ public void subscribe( String topic ) { processMsg( subMsg ); } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { - // change settings: + //TODO: change settings correctly: List topicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); StringBuilder stringBuilder = new StringBuilder(); for ( String t : topicsList ) { @@ -448,12 +450,13 @@ public void subscribe( String topic ) { if ( !topicsString.isEmpty() ) { topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); } + //TODO: synch over settings? this.settings.put( "topics", topicsString ); log.info( "not successful: {}", topic ); throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { - this.topics.put( topic, new AtomicLong( 0 ) ); - if ( collectionPerTopic.get() && !collectionExists( topic ) ) { + this.topicsMap.put( topic, new AtomicLong( 0 ) ); + if ( this.collectionPerTopic.get() && !collectionExists( topic ) ) { createNewCollection( topic ); } //TODO: rmv, only for debugging needed @@ -475,11 +478,11 @@ public void unsubscribe( List topics ) { public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - // change settings: + //TODO: synch over settings? this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); } else { - this.topics.remove( topic ); + this.topicsMap.remove( topic ); } } ); } @@ -492,14 +495,16 @@ void processMsg( Mqtt3Publish subMsg ) { String topic = subMsg.getTopic().toString(); String message = extractPayload( subMsg ); MqttMessage mqttMessage = new MqttMessage( message, topic ); - Long incrementedMsgCount = topics.get( topic ).incrementAndGet(); - topics.replace( topic, new AtomicLong( incrementedMsgCount ) ); + Long incrementedMsgCount = topicsMap.get( topic ).incrementAndGet(); + topicsMap.replace( topic, new AtomicLong( incrementedMsgCount ) ); addMessageToQueue( topic, message ); + //TODO: sync block if ( this.collectionPerTopic.get() ) { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); } else { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespace, getNamespaceId( this.namespace, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } + //TODO: end block MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? @@ -535,7 +540,8 @@ private boolean collectionExists( String collectionName ) { Catalog catalog = Catalog.getInstance(); Pattern pattern = new Pattern( collectionName ); List collectionList = null; - collectionList = catalog.getCollections( getNamespaceId( this.namespace, this.namespaceType), pattern ); + //TODO: synch Block + collectionList = catalog.getCollections( getNamespaceId( this.namespaceName, this.namespaceType), pattern ); return !collectionList.isEmpty(); } @@ -543,10 +549,15 @@ private boolean collectionExists( String collectionName ) { private void createNewCollection( String collectionName ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); + //TODO: synch Block: is this correct woth universal lock? + long namespaceID; + synchronized ( settingsLock ) { + namespaceID = getNamespaceId( this.namespaceName, this.namespaceType ); + } try { List dataStores = new ArrayList<>(); DdlManager.getInstance().createCollection( - getNamespaceId( this.namespace, this.namespaceType ), + namespaceID, collectionName, true, //only creates collection if it does not already exist. dataStores.size() == 0 ? null : dataStores, @@ -562,21 +573,24 @@ private void createNewCollection( String collectionName ) { private void createAllCollections() { - if ( this.collectionPerTopic.get() ) { - for ( String t : this.topics.keySet() ) { - if ( !collectionExists( t ) ) { - createNewCollection( t ); - } - } - } else { - if ( !(this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { - if ( !collectionExists( this.collectionName ) ) { - createNewCollection( this.collectionName ); + synchronized ( settingsLock ) { + if ( this.collectionPerTopic.get() ) { + for ( String t : this.topicsMap.keySet() ) { + if ( !collectionExists( t ) ) { + createNewCollection( t ); + } } } else { - throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + if ( !(this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { + if ( !collectionExists( this.collectionName ) ) { + createNewCollection( this.collectionName ); + } + } else { + throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + } } } + } @@ -703,25 +717,28 @@ public MonitoringPage() { public void update() { - /* TODO : rmv concurency trial + /* TODO : rmv concurency test String s = namespace; String c; for ( ;; ) { - if ( s.equals( namespace ) ) { - c = "!"; - } else { - c = "."; + synchronized ( namespace ) { + if ( s.equals( namespace ) ) { + c = "!"; + } else { + c = "."; + } } + System.out.print(c); } +*/ - */ topicsTable.reset(); - if ( topics.isEmpty() ) { + if ( topicsMap.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { - for ( Entry t : topics.entrySet() ) { + for ( Entry t : topicsMap.entrySet() ) { topicsTable.addRow( t.getKey(), t.getValue() ); } } From 54b0993b03b04796cdd9ec861e8cb2ed2f0909b8 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 30 Jul 2023 12:23:50 +0200 Subject: [PATCH 070/114] deleted unused methods --- .../org/polypheny/db/mqtt/StreamCapture.java | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 30dbaefd21..341ab597dd 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -58,33 +58,6 @@ public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { } - /** - * @return the id of the collection that was either already existing with the topic as name or that was newly created - */ -/* long getCollectionId() { - Catalog catalog = Catalog.getInstance(); - //check for collection with same name - List collectionList = catalog.getCollections( this.receivedMqttMessage.getNamespaceId(), null ); - for ( CatalogCollection collection : collectionList ) { - if ( collection.name.equals( this.receivedMqttMessage.getCollectionName() ) ) { - int queryInterfaceId = QueryInterfaceManager.getInstance().getQueryInterface( this.receivedMqttMessage.getUniqueNameOfInterface() ).getQueryInterfaceId(); - if ( !collection.placements.contains( queryInterfaceId ) ) { - log.info( "found matching collection!" ); - //TODO: Nur AdapterID können als Placements hinzugefügt werden. nicht QueryInterfaceIds -> Marco fragen. - //catalog.addCollectionPlacement( queryInterfaceId, collection.id, PlacementType.MANUAL ); - return collection.id; - } else { - log.info( "found matching collection!" ); - return collection.id; - } - } - } - - return 0; - } - */ - - // added by Datomo public void insertDocument( String collectionName) { String sqlCollectionName = this.receivedMqttMessage.getNamespaceName() + "." + collectionName; @@ -113,23 +86,6 @@ public void insertDocument( String collectionName) { } - public List getMessages( String namespaceName, String collectionName ) { - List listOfDocuments = scanCollection( namespaceName, collectionName ); - List listOfMessage = new ArrayList<>(); - for ( String document : listOfDocuments ) { - String[] documentAsList = document.split( "," ); - String topic = documentAsList[1].substring( documentAsList[0].indexOf( ':' ) ); - topic = topic.substring( topic.indexOf( '"' ) + 1, topic.lastIndexOf( '"' ) ); - - String message = documentAsList[2].trim(); - message = message.substring( message.indexOf( ':' ) ).trim(); - message = message.substring( message.indexOf( '"' ) + 1, message.lastIndexOf( '"' ) ); - listOfMessage.add( new MqttMessage( message, topic ) ); - } - return listOfMessage; - } - - // added by Datomo public List scanCollection( String namespaceName, String collectionName ) { String sqlCollectionName = namespaceName + "." + collectionName; From 8a5a9c8de679b54edf043e8028fa28be18cddb4c Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 7 Aug 2023 18:55:24 +0200 Subject: [PATCH 071/114] ssl code added but commented --- .../db/catalog/entity/CatalogCollection.java | 4 +- .../java/org/polypheny/db/ddl/DdlManager.java | 2 + .../db/schema/LogicalStreamCollection.java | 49 ++++++++++++ .../org/polypheny/db/ddl/DdlManagerImpl.java | 50 ++++++++++++ .../db/schema/PolySchemaBuilder.java | 9 +++ .../org/polypheny/db/catalog/CatalogImpl.java | 2 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 77 ++++++++++++++++--- 7 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/CatalogCollection.java b/core/src/main/java/org/polypheny/db/catalog/entity/CatalogCollection.java index 19b65f4d47..18b1690c3f 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/CatalogCollection.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/CatalogCollection.java @@ -61,13 +61,13 @@ public Serializable[] getParameterArray() { public CatalogCollection addPlacement( int adapterId ) { List placements = new ArrayList<>( this.placements ); placements.add( adapterId ); - return new CatalogCollection( databaseId, namespaceId, id, name, placements, EntityType.ENTITY, physicalName ); + return new CatalogCollection( databaseId, namespaceId, id, name, placements, entityType, physicalName ); } public CatalogCollection removePlacement( int adapterId ) { List placements = this.placements.stream().filter( id -> id != adapterId ).collect( Collectors.toList() ); - return new CatalogCollection( databaseId, namespaceId, id, name, placements, EntityType.ENTITY, physicalName ); + return new CatalogCollection( databaseId, namespaceId, id, name, placements, entityType, physicalName ); } diff --git a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java index 0e6741415f..4fd45b9f20 100644 --- a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java +++ b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java @@ -476,6 +476,8 @@ public static DdlManager getInstance() { public abstract void createCollection( long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException; + public abstract void createStreamCollection(long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException; + public abstract void addCollectionPlacement( long namespaceId, String name, List stores, Statement statement ); /** diff --git a/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java b/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java new file mode 100644 index 0000000000..8cb051faf7 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.schema; + +import java.util.List; +import org.polypheny.db.algebra.AlgNode; +import org.polypheny.db.algebra.core.Modify; +import org.polypheny.db.algebra.core.Modify.Operation; +import org.polypheny.db.algebra.type.AlgProtoDataType; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.prepare.Prepare.CatalogReader; +import org.polypheny.db.rex.RexNode; + + +public class LogicalStreamCollection extends LogicalTable { + + protected LogicalStreamCollection( long tableId, String logicalSchemaName, String logicalTableName, AlgProtoDataType protoRowType ) { + super( tableId, logicalSchemaName, logicalTableName, List.of( 0L ), List.of( "d" ), protoRowType, NamespaceType.DOCUMENT ); + } + + @Override + public Modify toModificationAlg( + AlgOptCluster cluster, + AlgOptTable table, + CatalogReader catalogReader, + AlgNode input, + Operation operation, + List updateColumnList, + List sourceExpressionList, + boolean flattened ) { + throw new RuntimeException( "Collection of type STREAM can not be modified." ); + } +} diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 338d484a69..094facf45e 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -2293,6 +2293,56 @@ public void createCollection( long schemaId, String name, boolean ifNotExists, L } + @Override + public void createStreamCollection( long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException { + name = adjustNameIfNeeded( name, schemaId ); + + if ( assertEntityExists( schemaId, name, ifNotExists ) ) { + return; + } + + if ( stores == null ) { + // Ask router on which store(s) the table should be placed + stores = RoutingManager.getInstance().getCreatePlacementStrategy().getDataStoresForNewTable(); + } + + long collectionId; + long partitionId; + try { + collectionId = catalog.addCollectionLogistics( schemaId, name, stores, false ); + partitionId = catalog.getPartitionGroups( collectionId ).get( 0 ).id; + } catch ( GenericCatalogException e ) { + throw new RuntimeException( e ); + } + + catalog.addCollection( + collectionId, + name, + schemaId, + statement.getPrepareContext().getCurrentUserId(), + EntityType.STREAM, + true ); + + // Initially create DataPlacement containers on every store the table should be placed. + CatalogCollection catalogCollection = catalog.getCollection( collectionId ); + + // Trigger rebuild of schema; triggers schema creation on adapters + PolySchemaBuilder.getInstance().getCurrent(); + + for ( DataStore store : stores ) { + catalog.addCollectionPlacement( + store.getAdapterId(), + catalogCollection.id, + PlacementType.AUTOMATIC ); + + afterDocumentLogistics( store, collectionId ); + + store.createCollection( statement.getPrepareContext(), catalogCollection, store.getAdapterId() ); + } + + } + + private boolean assertEntityExists( long namespaceId, String name, boolean ifNotExists ) throws EntityAlreadyExistsException { // Check if there is already an entity with this name if ( catalog.checkIfExistsEntity( namespaceId, name ) ) { diff --git a/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java b/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java index 5a09a8b0b6..825a417f2b 100644 --- a/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java +++ b/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java @@ -32,6 +32,7 @@ import org.polypheny.db.adapter.Adapter; import org.polypheny.db.adapter.AdapterManager; import org.polypheny.db.adapter.DataContext; +import org.polypheny.db.adapter.java.AbstractQueryableTable; import org.polypheny.db.algebra.logical.lpg.LogicalGraph; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeFactory; @@ -210,6 +211,14 @@ private void buildDocumentLogical( AbstractPolyphenyDbSchema polyphenyDbSchema, catalogEntity.getNamespaceName(), catalogEntity.name, AlgDataTypeImpl.proto( fieldInfo.build() ) ); + + } else if ( catalogEntity.entityType == EntityType.STREAM ) { + entity = new LogicalStreamCollection( + catalogEntity.id, + catalogEntity.getNamespaceName(), + catalogEntity.name, + AlgDataTypeImpl.proto( fieldInfo.build() ) ); + } else { throw new RuntimeException( "Unhandled table type: " + catalogEntity.entityType.name() ); } diff --git a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java index 275d43e49a..7bd5c33a95 100644 --- a/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java +++ b/plugins/mapdb-catalog/src/main/java/org/polypheny/db/catalog/CatalogImpl.java @@ -2420,7 +2420,7 @@ public long addCollection( Long id, String name, long schemaId, int currentUserI collectionId, name, List.of(), - EntityType.ENTITY, + entity, null ); synchronized ( this ) { diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 90d7e87e56..7a3a0b0c7b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,9 +19,19 @@ import com.google.common.collect.ImmutableList; import com.hivemq.client.mqtt.MqttClient; +import com.hivemq.client.mqtt.MqttClientSslConfig; +import com.hivemq.client.mqtt.MqttClientSslConfigBuilder; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.nio.charset.Charset; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -30,9 +40,10 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; @@ -69,6 +80,7 @@ import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; +import org.polypheny.db.util.PolyphenyHomeDirManager; public class MqttStreamPlugin extends Plugin { @@ -165,7 +177,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { - createNewCollection( this.collectionName ); + createStreamCollection( this.collectionName ); } } } @@ -173,19 +185,61 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au @Override public void run() { +/* + KeyStore keyStore = null; + try { + keyStore = KeyStore.getInstance("PKCS12"); + } catch ( KeyStoreException e ) { + throw new RuntimeException( e ); + } + //https://community.hivemq.com/t/sslwithdefaultconfig/946/4 + // load default jvm keystore + //todo: this.getUniqueName() + String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; + try { + keyStore.load(new FileInputStream( PolyphenyHomeDirManager.getInstance().getFileIfExists( path ) ), "Neha1512".toCharArray()); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } catch ( NoSuchAlgorithmException e ) { + throw new RuntimeException( e ); + } catch ( CertificateException e ) { + throw new RuntimeException( e ); + } + + TrustManagerFactory tmf = null; + try { + tmf = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + } catch ( NoSuchAlgorithmException e ) { + throw new RuntimeException( e ); + } + + try { + tmf.init(keyStore); + } catch ( KeyStoreException e ) { + throw new RuntimeException( e ); + } + + */ - // commented code used for SSL connection this.client = MqttClient.builder() .useMqttVersion3() .identifier( getUniqueName() ) .serverHost( broker ) .serverPort( brokerPort ) - //.useSslWithDefaultConfig() + /* + .sslConfig() + //.keyManagerFactory(kmf) + .trustManagerFactory(tmf) + .applySslConfig() + + */ + .useSslWithDefaultConfig() .buildAsync(); client.connectWith() //.simpleAuth() - //.username("my-user") + //.username("") //.password("my-password".getBytes()) //.applySimpleAuth() .send() @@ -403,7 +457,7 @@ protected void reloadSettings( List updatedSettings ) { if ( !mode ) { if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { if ( !collectionExists( newCollectionName ) ) { - createNewCollection( this.collectionName ); + createStreamCollection( this.collectionName ); } this.collectionName = newCollectionName; createAllCollections(); @@ -457,7 +511,7 @@ public void subscribe( String topic ) { } else { this.topicsMap.put( topic, new AtomicLong( 0 ) ); if ( this.collectionPerTopic.get() && !collectionExists( topic ) ) { - createNewCollection( topic ); + createStreamCollection( topic ); } //TODO: rmv, only for debugging needed log.info( "Successful subscription to topic:{}.", topic ); @@ -524,7 +578,7 @@ public List topicsToList( String topics ) { List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); for ( int i = 0; i < topicsList.size(); i++ ) { String topic = topicsList.get( i ).trim(); - if ( !topic.isEmpty() ) { + if ( !topic.isBlank() ) { topicsList.set( i, topic ); } } @@ -546,7 +600,7 @@ private boolean collectionExists( String collectionName ) { } - private void createNewCollection( String collectionName ) { + private void createStreamCollection( String collectionName ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); //TODO: synch Block: is this correct woth universal lock? @@ -556,6 +610,7 @@ private void createNewCollection( String collectionName ) { } try { List dataStores = new ArrayList<>(); + //TODO: StreamCollection einbinden DdlManager.getInstance().createCollection( namespaceID, collectionName, @@ -577,13 +632,13 @@ private void createAllCollections() { if ( this.collectionPerTopic.get() ) { for ( String t : this.topicsMap.keySet() ) { if ( !collectionExists( t ) ) { - createNewCollection( t ); + createStreamCollection( t ); } } } else { if ( !(this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { if ( !collectionExists( this.collectionName ) ) { - createNewCollection( this.collectionName ); + createStreamCollection( this.collectionName ); } } else { throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); From 76c8e349e39653d98596cbe390c90da316ab607e Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 7 Aug 2023 19:03:32 +0200 Subject: [PATCH 072/114] comment ssl conn code --- .../src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 7a3a0b0c7b..89ecba7400 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -232,9 +232,9 @@ public void run() { //.keyManagerFactory(kmf) .trustManagerFactory(tmf) .applySslConfig() + .useSslWithDefaultConfig() */ - .useSslWithDefaultConfig() .buildAsync(); client.connectWith() From 15687e2d2a77d12eea541f70731bdeb7ced93fd0 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 7 Aug 2023 19:16:34 +0200 Subject: [PATCH 073/114] integrated universal lock for concurrency --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 220 +++++++++--------- 1 file changed, 111 insertions(+), 109 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 89ecba7400..19020bf2ae 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -329,87 +329,88 @@ private long createNamespace( String namespaceName, NamespaceType namespaceType @Override protected void reloadSettings( List updatedSettings ) { - //TODO: synch lock araound for schleife: - for ( String changedSetting : updatedSettings ) { - switch ( changedSetting ) { - case "topics": - List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); - List topicsToSub = new ArrayList<>(); - for ( String newTopic : newTopicsList ) { - if ( !topicsMap.containsKey( newTopic ) ) { - topicsToSub.add( newTopic ); + synchronized ( settingsLock ) { + + for ( String changedSetting : updatedSettings ) { + switch ( changedSetting ) { + case "topics": + List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); + List topicsToSub = new ArrayList<>(); + for ( String newTopic : newTopicsList ) { + if ( !topicsMap.containsKey( newTopic ) ) { + topicsToSub.add( newTopic ); + } } - } - if ( !topicsToSub.isEmpty() ) { - subscribe( topicsToSub ); - } - List topicsToUnsub = new ArrayList<>(); - for ( String oldTopic : topicsMap.keySet() ) { - if ( !newTopicsList.contains( oldTopic ) ) { - topicsToUnsub.add( oldTopic ); + if ( !topicsToSub.isEmpty() ) { + subscribe( topicsToSub ); + } + List topicsToUnsub = new ArrayList<>(); + for ( String oldTopic : topicsMap.keySet() ) { + if ( !newTopicsList.contains( oldTopic ) ) { + topicsToUnsub.add( oldTopic ); + } + } + if ( !topicsToUnsub.isEmpty() ) { + unsubscribe( topicsToUnsub ); + } + break; + + case "namespace": + /* + try { + this.wait(2000); + } catch ( InterruptedException e ) { + throw new RuntimeException( e ); + } + this.notify(); + + */ + //TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! + /* + try { + client.wait(); + } catch ( InterruptedException e ) { + throw new RuntimeException( e ); } - } - if ( !topicsToUnsub.isEmpty() ) { - unsubscribe( topicsToUnsub ); - } - break; - - case "namespace": - /* - try { - this.wait(2000); - } catch ( InterruptedException e ) { - throw new RuntimeException( e ); - } - this.notify(); - - */ -//TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! - /* - try { - client.wait(); - } catch ( InterruptedException e ) { - throw new RuntimeException( e ); - } - */ - synchronized ( this.namespaceName ) { - String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - if ( updatedSettings.contains( "namespace type" ) ) { - if ( updatedSettings.indexOf( "namespace type" ) < updatedSettings.indexOf( "namespace" ) ) { - NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + */ + synchronized ( this.namespaceName ) { + String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); + if ( updatedSettings.contains( "namespace type" ) ) { + if ( updatedSettings.indexOf( "namespace type" ) < updatedSettings.indexOf( "namespace" ) ) { + NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + try { + if ( !namespaceExists( newNamespaceName, type ) ) { + createNamespace( newNamespaceName, type ); + } + this.namespaceName = newNamespaceName; + this.namespaceType = type; + createAllCollections(); + } catch ( RuntimeException e ) { + this.settings.put( "namespace", this.namespaceName ); + this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + throw new RuntimeException( e ); + } + } // else checking for namespace happens in case "namespace type" + } else { try { - if ( !namespaceExists( newNamespaceName, type ) ) { - createNamespace( newNamespaceName, type ); + if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { + createNamespace( newNamespaceName, this.namespaceType ); } this.namespaceName = newNamespaceName; - this.namespaceType = type; createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); - this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } - } // else checking for namespace happens in case "namespace type" - } else { - try { - if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { - createNamespace( newNamespaceName, this.namespaceType ); - } - this.namespaceName = newNamespaceName; - createAllCollections(); - } catch ( RuntimeException e ) { - this.settings.put( "namespace", this.namespaceName ); - throw new RuntimeException( e ); } } - } - break; + break; - case "namespace type": - NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); - //synchronized ( this.namespaceType ) { + case "namespace type": + NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + //synchronized ( this.namespaceType ) { if ( updatedSettings.contains( "namespace" ) ) { if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespace type" ) ) { String newName = this.getCurrentSettings().get( "namespace" ); @@ -439,37 +440,38 @@ protected void reloadSettings( List updatedSettings ) { } } - // } - - break; - case "collection per topic": - this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ) ); - createAllCollections(); - break; - case "collection name": - String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); - boolean mode; - if ( updatedSettings.contains( "collection per topic" ) ) { - mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); - } else { - mode = this.collectionPerTopic.get(); - } - if ( !mode ) { - if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { - if ( !collectionExists( newCollectionName ) ) { - createStreamCollection( this.collectionName ); - } - this.collectionName = newCollectionName; - createAllCollections(); + // } + + break; + case "collection per topic": + this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ) ); + createAllCollections(); + break; + case "collection name": + String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); + boolean mode; + if ( updatedSettings.contains( "collection per topic" ) ) { + mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); } else { - this.settings.put( "collection name", this.collectionName ); - throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + mode = this.collectionPerTopic.get(); } + if ( !mode ) { + if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { + if ( !collectionExists( newCollectionName ) ) { + createStreamCollection( this.collectionName ); + } + this.collectionName = newCollectionName; + createAllCollections(); + } else { + this.settings.put( "collection name", this.collectionName ); + throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + } - } else { - this.collectionName = newCollectionName; - } - break; + } else { + this.collectionName = newCollectionName; + } + break; + } } } } @@ -504,8 +506,9 @@ public void subscribe( String topic ) { if ( !topicsString.isEmpty() ) { topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); } - //TODO: synch over settings? - this.settings.put( "topics", topicsString ); + synchronized ( settingsLock ) { + this.settings.put( "topics", topicsString ); + } log.info( "not successful: {}", topic ); throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { @@ -513,8 +516,6 @@ public void subscribe( String topic ) { if ( this.collectionPerTopic.get() && !collectionExists( topic ) ) { createStreamCollection( topic ); } - //TODO: rmv, only for debugging needed - log.info( "Successful subscription to topic:{}.", topic ); } } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). @@ -532,8 +533,9 @@ public void unsubscribe( List topics ) { public void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { - //TODO: synch over settings? - this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); + synchronized ( settingsLock ) { + this.settings.put( "topics", this.settings.get( "topics" ) + "," + topic ); + } throw new RuntimeException( "Topic " + topic + " could not be unsubscribed.", throwable ); } else { this.topicsMap.remove( topic ); @@ -552,13 +554,13 @@ void processMsg( Mqtt3Publish subMsg ) { Long incrementedMsgCount = topicsMap.get( topic ).incrementAndGet(); topicsMap.replace( topic, new AtomicLong( incrementedMsgCount ) ); addMessageToQueue( topic, message ); - //TODO: sync block - if ( this.collectionPerTopic.get() ) { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); - } else { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + synchronized ( settingsLock ) { + if ( this.collectionPerTopic.get() ) { + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); + } else { + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + } } - //TODO: end block MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); String content = streamProcessor.processStream(); //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? @@ -594,8 +596,9 @@ private boolean collectionExists( String collectionName ) { Catalog catalog = Catalog.getInstance(); Pattern pattern = new Pattern( collectionName ); List collectionList = null; - //TODO: synch Block - collectionList = catalog.getCollections( getNamespaceId( this.namespaceName, this.namespaceType), pattern ); + synchronized ( settingsLock ) { + collectionList = catalog.getCollections( getNamespaceId( this.namespaceName, this.namespaceType ), pattern ); + } return !collectionList.isEmpty(); } @@ -603,7 +606,6 @@ private boolean collectionExists( String collectionName ) { private void createStreamCollection( String collectionName ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - //TODO: synch Block: is this correct woth universal lock? long namespaceID; synchronized ( settingsLock ) { namespaceID = getNamespaceId( this.namespaceName, this.namespaceType ); From 4ea38ff5fa9480fa9f439a858545523ca79a3070 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 31 Aug 2023 18:31:18 +0200 Subject: [PATCH 074/114] started testing environment --- plugins/mqtt-stream/build.gradle | 24 ++++++++--- .../java/org/polypheny/db/mqtt/MqttTest.java | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index c0fd28c93c..80cdc02fc8 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -1,8 +1,14 @@ group "org.polypheny" +configurations { + tests { + extendsFrom testRuntimeOnly + } +} dependencies { compileOnly project(":core") + compileOnly project(":dbms") compileOnly project(":monitoring") compileOnly project(":plugins:mql-language") @@ -12,21 +18,27 @@ dependencies { implementation group: "org.mongodb", name: "mongodb-driver-sync", version: mongodb_driver_sync_version // Apache 2.0 // --- Test Compile --- + testImplementation project(path: ":dbms", configuration: "test") + testImplementation project(path: ":dbms") testImplementation project(path: ":core", configuration: "tests") testImplementation project(path: ":core") testCompileOnly group: 'org.pf4j', name: 'pf4j', version: pf4jVersion testImplementation group: "junit", name: "junit", version: junit_version + + // --- Test Compile --- + } sourceSets { main { java { - srcDirs = ["src/main/java"] + srcDirs = ["src/main/java", "build/generated-sources"] outputDir = file(project.buildDir.absolutePath + "/classes") } resources { srcDirs = ["src/main/resources"] + exclude "version/*.properties" } output.resourcesDir = file(project.buildDir.absolutePath + "/classes") } @@ -36,7 +48,8 @@ sourceSets { outputDir = file(project.buildDir.absolutePath + "/test-classes") } resources { - srcDirs = ["src/test/resources"] + // We need the main resources for the tests as well. + srcDirs = ["src/test/resources", "src/main/resources"] } output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") } @@ -64,9 +77,10 @@ jar { attributes "Version": "$project.version" } } -java { - withJavadocJar() - withSourcesJar() + +java{ + withJavadocJar(); + withSourcesJar(); } licensee { diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java new file mode 100644 index 0000000000..25348b006c --- /dev/null +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; + +public class MqttTest { + @BeforeClass + public static void init() { + TestHelper.getInstance(); + + } + @Test + public void algTest() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "db.buttontest.find({value:10})"; + MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + streamProcessor.processStream(); + } +} From af47b78a22126e26e5363873049c8f7928004c36 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 3 Sep 2023 21:09:53 +0200 Subject: [PATCH 075/114] StreamProcessor in progress --- .../document/LogicalDocumentValues.java | 1 + .../processing/LogicalAlgAnalyzeShuttle.java | 6 + .../polypheny/db/stream/StreamProcessor.java | 2 + .../db/routing/routers/AbstractDqlRouter.java | 2 + .../db/stream/StreamProcessorImpl.java | 7 +- .../languages/mql2alg/MqlToAlgConverter.java | 51 ++ .../polypheny/db/mqtt/MqttStreamPlugin.java | 378 +++++++++---- .../db/mqtt/MqttStreamProcessor.java | 522 +++++++++++++++++- .../db/mqtt/ReceivedMqttMessage.java | 12 +- 9 files changed, 841 insertions(+), 140 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java b/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java index bb5bf83111..524569295f 100644 --- a/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java +++ b/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java @@ -38,6 +38,7 @@ import org.polypheny.db.plan.AlgTraitSet; import org.polypheny.db.plan.Convention; import org.polypheny.db.rex.RexLiteral; +import org.polypheny.db.schema.ModelTrait; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFactoryImpl; import org.polypheny.db.util.BsonUtil; diff --git a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java index 1ba3b3de60..1e75b09b79 100644 --- a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java +++ b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java @@ -38,6 +38,7 @@ import org.polypheny.db.algebra.logical.document.LogicalDocumentScan; import org.polypheny.db.algebra.logical.document.LogicalDocumentSort; import org.polypheny.db.algebra.logical.document.LogicalDocumentTransformer; +import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; import org.polypheny.db.algebra.logical.lpg.LogicalGraph; import org.polypheny.db.algebra.logical.lpg.LogicalLpgAggregate; import org.polypheny.db.algebra.logical.lpg.LogicalLpgFilter; @@ -261,6 +262,11 @@ public AlgNode visit( LogicalDocumentScan scan ) { return super.visit( scan ); } + public AlgNode visit( LogicalDocumentValues values ) { + hashBasis.add( "LogicalDocumentValues" ); + return super.visit( values ); + } + @Override public AlgNode visit( LogicalDocumentSort sort ) { diff --git a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java index e81bdd6d1e..2eb3681235 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java @@ -24,4 +24,6 @@ public interface StreamProcessor { //TODO: maybe change type to MqttStream? String getStream( ); + boolean validateContent( String stream ); + } diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/AbstractDqlRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/AbstractDqlRouter.java index 93b549f3b7..25ae8c266e 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/AbstractDqlRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/AbstractDqlRouter.java @@ -37,6 +37,7 @@ import org.polypheny.db.algebra.core.lpg.LpgAlg.NodeType; import org.polypheny.db.algebra.logical.common.LogicalTransformer; import org.polypheny.db.algebra.logical.document.LogicalDocumentScan; +import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; import org.polypheny.db.algebra.logical.lpg.LogicalLpgScan; import org.polypheny.db.algebra.logical.relational.LogicalModify; import org.polypheny.db.algebra.logical.relational.LogicalScan; @@ -172,6 +173,7 @@ public AlgNode routeDocument( RoutedAlgBuilder builder.push( handleDocumentScan( (DocumentScan) alg, statement, builder, null ).build() ); return alg; } else if ( alg.getDocType() == DocType.VALUES ) { + builder.push( handleDocuments( (LogicalDocumentValues) alg, builder ).build() ); return alg; } throw new UnsupportedOperationException(); diff --git a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java index 25805f292b..56167eee15 100644 --- a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java @@ -25,10 +25,15 @@ public StreamProcessorImpl(String stream) { this.stream = stream; } - @Override public String getStream() { return stream; } + + @Override + public boolean validateContent( String stream ) { + return false; + } + } diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java index 50a0fb262b..5fbc6f61b7 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java @@ -328,6 +328,57 @@ public AlgRoot convert( MqlCollectionStatement query ) { } + /** + * Converts the initial MongoQl by stepping through it iteratively + * but queries on a given input instead of a given collection + * @param query the query in MqlNode format + * @param input the value that should be queried + * @return the {@link AlgNode} format of the initial query + */ + public AlgRoot convert( MqlCollectionStatement query, AlgNode input ) { + Type kind = query.getMqlKind(); + this.usesDocumentModel = true; + + AlgDataType rowType = input.getRowType(); + + this.builder = new RexBuilder( cluster.getTypeFactory() ); + + AlgRoot root; + + switch ( kind ) { + case FIND: + AlgNode find = convertFind( (MqlFind) query, rowType, input ); + root = AlgRoot.of( find, find.getRowType(), Kind.SELECT ); + break; + case COUNT: + AlgNode count = convertCount( (MqlCount) query, rowType, input ); + root = AlgRoot.of( count, count.getRowType(), Kind.SELECT ); + break; + case AGGREGATE: + AlgNode aggregate = convertAggregate( (MqlAggregate) query, rowType, input ); + root = AlgRoot.of( aggregate, Kind.SELECT ); + break; + /// dmls + case INSERT: + root = AlgRoot.of( convertInsert( (MqlInsert) query, entity ), Kind.INSERT ); + break; + case DELETE: + case FIND_DELETE: + root = AlgRoot.of( convertDelete( (MqlDelete) query, entity, input ), Kind.DELETE ); + break; + case UPDATE: + root = AlgRoot.of( convertUpdate( (MqlUpdate) query, entity, input ), Kind.UPDATE ); + break; + default: + throw new IllegalStateException( "Unexpected value: " + kind ); + } + /*if ( usesDocumentModel ) { + root.usesDocumentModel = true; + }*/ + return root; + } + + private AlgOptTable getEntity( MqlCollectionStatement query, String dbSchemaName ) { List names = ImmutableList.of( dbSchemaName, query.getCollection() ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 19020bf2ae..68000001d7 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,13 +19,10 @@ import com.google.common.collect.ImmutableList; import com.hivemq.client.mqtt.MqttClient; -import com.hivemq.client.mqtt.MqttClientSslConfig; -import com.hivemq.client.mqtt.MqttClientSslConfigBuilder; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; import java.security.KeyStore; @@ -37,12 +34,10 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -101,10 +96,11 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - mqttDefaultSettings.put( "namespace", "namespace1" ); - mqttDefaultSettings.put( "namespace type", "DOCUMENT" ); - mqttDefaultSettings.put( "collection per topic", "TRUE" ); - mqttDefaultSettings.put( "collection name", "default" ); + mqttDefaultSettings.put( "namespace", "wohnzimmer" ); + mqttDefaultSettings.put( "namespaceType", "DOCUMENT" ); + mqttDefaultSettings.put( "collectionPerTopic", "TRUE" ); + mqttDefaultSettings.put( "collectionName", "default" ); + mqttDefaultSettings.put( "Query Interface Name", "mqtt" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -125,14 +121,16 @@ public static class MqttStreamServer extends QueryInterface { public static final String INTERFACE_DESCRIPTION = "Connection establishment to a MQTT broker."; @SuppressWarnings("WeakerAccess") public static final List AVAILABLE_SETTINGS = ImmutableList.of( - new QueryInterfaceSettingString( "broker", false, true, false, "localhost" ), - new QueryInterfaceSettingInteger( "brokerPort", false, true, false, 1883 ), - new QueryInterfaceSettingString( "namespace", false, true, true, "namespace1" ), + new QueryInterfaceSettingString( "broker", false, true, false, null ), + new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), + new QueryInterfaceSettingString( "namespace", false, true, true, null ), // "RELATIONAL", "GRAPH" type are not supported yet. - new QueryInterfaceSettingList( "namespace type", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), - new QueryInterfaceSettingList( "collection per topic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), - new QueryInterfaceSettingString( "collection name", true, false, true, null ), - new QueryInterfaceSettingString( "topics", false, true, true, null ) + new QueryInterfaceSettingList( "namespaceType", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), + new QueryInterfaceSettingList( "collectionPerTopic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), + new QueryInterfaceSettingString( "collectionName", true, false, true, null ), + new QueryInterfaceSettingString( "topics", false, true, true, null ), + new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), + new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); @Getter @@ -140,6 +138,13 @@ public static class MqttStreamServer extends QueryInterface { @Getter private final int brokerPort; private Map topicsMap = new ConcurrentHashMap<>(); + /** + * Contains the predicates that determines whether a message is inserted. + * The Predicates are applied to the message in form of a filter and if this query returns true,the message will + * be inserted. If there is no predicate for a topic, then all messages are saved. + * If there are several queries for the topic of the message, they are comma seperated. + */ + private Map filterMap = new ConcurrentHashMap<>(); private ConcurrentLinkedQueue messageQueue = new ConcurrentLinkedQueue<>(); private Mqtt3AsyncClient client; private String namespaceName; @@ -148,6 +153,8 @@ public static class MqttStreamServer extends QueryInterface { private String collectionName; private final long databaseId; private final int userId; + final boolean ssl; + boolean createCommonCollection = false; private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -158,84 +165,100 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.monitoringPage = new MonitoringPage(); this.broker = settings.get( "broker" ).trim(); this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ).trim() ); - this.databaseId = Catalog.defaultDatabaseId; this.userId = Catalog.defaultUserId; String name = settings.get( "namespace" ).trim(); - NamespaceType type = NamespaceType.valueOf( settings.get( "namespace type" ) ); + NamespaceType type = NamespaceType.valueOf( settings.get( "namespaceType" ) ); + if ( type != NamespaceType.DOCUMENT ) { + throw new RuntimeException( "Namespace types other than the DOCUMENT type are not yet supported." ); + } if ( !namespaceExists( name, type ) ) { createNamespace( name, type ); } this.namespaceName = name; this.namespaceType = type; - - this.collectionPerTopic = new AtomicBoolean(Boolean.parseBoolean( settings.get( "collection per topic" ) ) ); - this.collectionName = settings.get( "collection name" ).trim(); + this.collectionPerTopic = new AtomicBoolean( Boolean.parseBoolean( settings.get( "collectionPerTopic" ) ) ); + this.collectionName = settings.get( "collectionName" ) == null ? settings.get( "collectionName" ) : settings.get( "collectionName" ).trim(); if ( !this.collectionPerTopic.get() ) { - if ( (this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { - throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + if ( (this.collectionName.equals( null ) | this.collectionName.equals( "" )) ) { + throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { + this.createCommonCollection = true; createStreamCollection( this.collectionName ); } } + String queryString = settings.get( "filterQuery" ); + if ( queryString != null && !queryString.isBlank() ) { + saveQueriesInMap( queryString ); + } + + this.ssl = Boolean.parseBoolean( settings.get( "Tsl/SslConnection" ) ); + } @Override public void run() { -/* - KeyStore keyStore = null; - try { - keyStore = KeyStore.getInstance("PKCS12"); - } catch ( KeyStoreException e ) { - throw new RuntimeException( e ); - } - //https://community.hivemq.com/t/sslwithdefaultconfig/946/4 - // load default jvm keystore - //todo: this.getUniqueName() - String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; - try { - keyStore.load(new FileInputStream( PolyphenyHomeDirManager.getInstance().getFileIfExists( path ) ), "Neha1512".toCharArray()); - } catch ( IOException e ) { - throw new RuntimeException( e ); - } catch ( NoSuchAlgorithmException e ) { - throw new RuntimeException( e ); - } catch ( CertificateException e ) { - throw new RuntimeException( e ); - } + if ( ssl ) { + //TODO: look at book: essentials + KeyStore keyStore = null; + try { + keyStore = KeyStore.getInstance( "PKCS12" ); + } catch ( KeyStoreException e ) { + throw new RuntimeException( e ); + } + //https://community.hivemq.com/t/sslwithdefaultconfig/946/4 + // load default jvm keystore + //todo: this.getUniqueName() + String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; + try { + keyStore.load( new FileInputStream( PolyphenyHomeDirManager.getInstance().getFileIfExists( path ) ), "".toCharArray() ); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } catch ( NoSuchAlgorithmException e ) { + throw new RuntimeException( e ); + } catch ( CertificateException e ) { + throw new RuntimeException( e ); + } - TrustManagerFactory tmf = null; - try { - tmf = TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm()); - } catch ( NoSuchAlgorithmException e ) { - throw new RuntimeException( e ); - } + TrustManagerFactory tmf = null; + try { + tmf = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm() ); + } catch ( NoSuchAlgorithmException e ) { + throw new RuntimeException( e ); + } - try { - tmf.init(keyStore); - } catch ( KeyStoreException e ) { - throw new RuntimeException( e ); - } + try { + tmf.init( keyStore ); + } catch ( KeyStoreException e ) { + throw new RuntimeException( e ); + } - */ + this.client = MqttClient.builder() + .useMqttVersion3() + .identifier( getUniqueName() ) + .serverHost( broker ) + .serverPort( brokerPort ) - this.client = MqttClient.builder() - .useMqttVersion3() - .identifier( getUniqueName() ) - .serverHost( broker ) - .serverPort( brokerPort ) - /* - .sslConfig() + .sslConfig() //.keyManagerFactory(kmf) - .trustManagerFactory(tmf) + .trustManagerFactory( tmf ) .applySslConfig() - .useSslWithDefaultConfig() + .useSslWithDefaultConfig() - */ - .buildAsync(); + .buildAsync(); + + } else { + this.client = MqttClient.builder() + .useMqttVersion3() + .identifier( getUniqueName() ) + .serverHost( broker ) + .serverPort( brokerPort ) + .buildAsync(); + } client.connectWith() //.simpleAuth() @@ -248,10 +271,11 @@ public void run() { throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - subscribe( topicsToList( this.settings.get( "topics" ) ) ); + subscribe( toList( this.settings.get( "topics" ) ) ); } } ); + } @@ -334,7 +358,7 @@ protected void reloadSettings( List updatedSettings ) { for ( String changedSetting : updatedSettings ) { switch ( changedSetting ) { case "topics": - List newTopicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); + List newTopicsList = toList( this.getCurrentSettings().get( "topics" ) ); List topicsToSub = new ArrayList<>(); for ( String newTopic : newTopicsList ) { if ( !topicsMap.containsKey( newTopic ) ) { @@ -365,7 +389,7 @@ protected void reloadSettings( List updatedSettings ) { this.notify(); */ - //TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! + //TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! /* try { client.wait(); @@ -377,9 +401,9 @@ protected void reloadSettings( List updatedSettings ) { */ synchronized ( this.namespaceName ) { String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - if ( updatedSettings.contains( "namespace type" ) ) { - if ( updatedSettings.indexOf( "namespace type" ) < updatedSettings.indexOf( "namespace" ) ) { - NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + if ( updatedSettings.contains( "namespaceType" ) ) { + if ( updatedSettings.indexOf( "namespaceType" ) < updatedSettings.indexOf( "namespace" ) ) { + NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespaceType" ) ); try { if ( !namespaceExists( newNamespaceName, type ) ) { createNamespace( newNamespaceName, type ); @@ -389,10 +413,10 @@ protected void reloadSettings( List updatedSettings ) { createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); - this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } - } // else checking for namespace happens in case "namespace type" + } // else checking for namespace happens in case "namespaceType" } else { try { if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { @@ -408,11 +432,11 @@ protected void reloadSettings( List updatedSettings ) { } break; - case "namespace type": - NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespace type" ) ); + case "namespaceType": + NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespaceType" ) ); //synchronized ( this.namespaceType ) { if ( updatedSettings.contains( "namespace" ) ) { - if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespace type" ) ) { + if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespaceType" ) ) { String newName = this.getCurrentSettings().get( "namespace" ); try { if ( !namespaceExists( newName, newNamespaceType ) ) { @@ -423,7 +447,7 @@ protected void reloadSettings( List updatedSettings ) { createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); - this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } } // else checking for namespace happens in case "namespace" @@ -435,7 +459,7 @@ protected void reloadSettings( List updatedSettings ) { this.namespaceType = newNamespaceType; createAllCollections(); } catch ( RuntimeException e ) { - this.settings.put( "namespace type", String.valueOf( this.namespaceType ) ); + this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } } @@ -443,15 +467,15 @@ protected void reloadSettings( List updatedSettings ) { // } break; - case "collection per topic": - this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ) ); + case "collectionPerTopic": + this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collectionPerTopic" ) ) ); createAllCollections(); break; - case "collection name": - String newCollectionName = this.getCurrentSettings().get( "collection name" ).trim(); + case "collectionName": + String newCollectionName = this.getCurrentSettings().get( "collectionName" ).trim(); boolean mode; - if ( updatedSettings.contains( "collection per topic" ) ) { - mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collection per topic" ) ); + if ( updatedSettings.contains( "collectionPerTopic" ) ) { + mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collectionPerTopic" ) ); } else { mode = this.collectionPerTopic.get(); } @@ -463,14 +487,46 @@ protected void reloadSettings( List updatedSettings ) { this.collectionName = newCollectionName; createAllCollections(); } else { - this.settings.put( "collection name", this.collectionName ); - throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + this.settings.put( "collectionName", this.collectionName ); + throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } } else { this.collectionName = newCollectionName; } break; + case "filterQuery": + //TODO: Problem mit Komma in Query und trennen der Queries nach komma + String queryString = this.getCurrentSettings().get( "filterQuery" ); + if ( queryString == null || queryString.isBlank() ) { + filterMap.clear(); + } else { + String[] queryArray = queryString.split( "," ); + for ( String query : queryArray ) { + int separatorIndex = query.indexOf( ":" ); + String topic = query.substring( 0, separatorIndex ).trim(); + String mqlQuery = query.substring( separatorIndex + 1 ).trim(); + if ( filterMap.containsKey( topic ) && !filterMap.get( topic ).equals( mqlQuery ) ) { + filterMap.replace( topic, mqlQuery ); + } else { + filterMap.put( topic, mqlQuery ); + } + } + + for ( Entry entry : filterMap.entrySet() ) { + boolean remove = true; + for ( String query : queryArray ) { + if ( query.startsWith( entry.getKey() ) ) { + remove = false; + break; + } + } + if ( remove ) { + filterMap.remove( entry.getKey(), entry.getValue() ); + } + } + } + break; } } } @@ -495,7 +551,7 @@ public void subscribe( String topic ) { } ).send().whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { //TODO: change settings correctly: - List topicsList = topicsToList( this.getCurrentSettings().get( "topics" ) ); + List topicsList = toList( this.getCurrentSettings().get( "topics" ) ); StringBuilder stringBuilder = new StringBuilder(); for ( String t : topicsList ) { if ( !t.equals( topic ) ) { @@ -503,7 +559,7 @@ public void subscribe( String topic ) { } } String topicsString = stringBuilder.toString(); - if ( !topicsString.isEmpty() ) { + if ( topicsString != null && !topicsString.isBlank() ) { topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); } synchronized ( settingsLock ) { @@ -547,44 +603,115 @@ public void unsubscribe( String topic ) { void processMsg( Mqtt3Publish subMsg ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); - ReceivedMqttMessage receivedMqttMessage; + String topic = subMsg.getTopic().toString(); String message = extractPayload( subMsg ); + String wildcardTopic = ""; + if ( !topicsMap.containsKey( topic ) ) { + wildcardTopic = compareWithWildcardsTopic( topic ); + topicsMap.get( wildcardTopic ).incrementAndGet(); + } else { + topicsMap.get( topic ).incrementAndGet(); + } MqttMessage mqttMessage = new MqttMessage( message, topic ); - Long incrementedMsgCount = topicsMap.get( topic ).incrementAndGet(); - topicsMap.replace( topic, new AtomicLong( incrementedMsgCount ) ); addMessageToQueue( topic, message ); + if ( this.filterMap.containsKey( topic ) ) { + String filterQuery = this.filterMap.get( topic ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); + // false is returned when message should not be stored in DB + if ( streamProcessor.processStream() ) { + insert(mqttMessage, transaction); + } + } else { + insert(mqttMessage, transaction); + } + } + + + private void insert ( MqttMessage mqttMessage, Transaction transaction) { + //TODO: besserer Name für message objekt: was mit insert + ReceivedMqttMessage receivedMqttMessage; synchronized ( settingsLock ) { if ( this.collectionPerTopic.get() ) { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, topic ); + String collectionToBeSaved; + /* + if ( wildcardTopic != null || wildcardTopic.isBlank() ) { + + } else { + collectionToBeSaved = wildcardTopic; + } + */ + collectionToBeSaved = mqttMessage.getTopic(); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); } else { receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } } - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( receivedMqttMessage, statement ); - String content = streamProcessor.processStream(); - //TODO: what is returned from processStream if this Stream is not valid/ not result of filter...? - if ( !Objects.equals( content, "" ) ) { - StreamCapture streamCapture = new StreamCapture( getTransaction() ); - streamCapture.handleContent( receivedMqttMessage ); - } + + StreamCapture streamCapture = new StreamCapture( transaction ); + streamCapture.insert( receivedMqttMessage ); } + // helper methods: private static String extractPayload( Mqtt3Publish subMsg ) { return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); } - public List topicsToList( String topics ) { - List topicsList = new ArrayList<>( List.of( topics.split( "," ) ) ); - for ( int i = 0; i < topicsList.size(); i++ ) { - String topic = topicsList.get( i ).trim(); + /** + * format of queries comma seperated: :, :, ... + * + * @param queries + */ + private void saveQueriesInMap( String queries ) { + List queriesList = toList( queries ); + for ( String topicQueryString : queriesList ) { + int separatorIndex = topicQueryString.indexOf( ":" ); + String topic = topicQueryString.substring( 0, separatorIndex ); + String query = topicQueryString.substring( separatorIndex + 1 ); + if ( this.filterMap.containsKey( topic ) ) { + String val = this.filterMap.get( topic ); + // TODO: check: now also or can be done in query so no several queries for one topic: replace old query + this.filterMap.replace( topic, query ); + } else { + this.filterMap.put( topic, query ); + } + } + } + + + private String compareWithWildcardsTopic( String topic ) { + for ( String t : topicsMap.keySet() ) { + //multilevel wildcard + if ( t.contains( "#" ) && topic.startsWith( t.substring( 0, t.indexOf( "#" ) ) ) ) { + return t; + + } + // single level wildcard + if ( t.contains( "+" ) && topic.startsWith( t.substring( 0, t.indexOf( "+" ) ) ) && topic.endsWith( t.substring( t.indexOf( "+" ) + 1 ) ) ) { + return t; + } + } + return topic; + } + + + /** + * separates a string by commas and inserts the separated parts to a list. + * + * @param string + * @return List of seperated string values + */ + public List toList( String string ) { + List list = new ArrayList<>( List.of( string.split( "," ) ) ); + for ( int i = 0; i < list.size(); i++ ) { + String topic = list.get( i ).trim(); if ( !topic.isBlank() ) { - topicsList.set( i, topic ); + list.set( i, topic ); } } - return topicsList; + return list; } @@ -643,7 +770,7 @@ private void createAllCollections() { createStreamCollection( this.collectionName ); } } else { - throw new NullPointerException( "Collection per topic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); } } } @@ -706,13 +833,13 @@ public MonitoringPage() { InformationManager im = InformationManager.getInstance(); informationPage = new InformationPage( getUniqueName(), INTERFACE_NAME ).setLabel( "Interfaces" ); + informationPage.setRefreshFunction( this::update ); im.addPage( informationPage ); InformationGroup informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); brokerKv = new InformationKeyValue( informationGroupInfo ); im.registerInformation( brokerKv ); - informationGroupInfo.setRefreshFunction( this::update ); informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); im.addGroup( informationGroupTopics ); @@ -721,23 +848,24 @@ public MonitoringPage() { List.of( "Topic", "Number of received messages" ) ); im.registerInformation( topicsTable ); - informationGroupTopics.setRefreshFunction( this::update ); - InformationGroup informationGroupMessage = new InformationGroup( informationPage, "Recently arrived messages" ).setOrder( 2 ); + InformationGroup informationGroupMessage = new InformationGroup( informationPage, "Recently received messages" ).setOrder( 2 ); im.addGroup( informationGroupMessage ); messageTable = new InformationTable( informationGroupMessage, List.of( "Topic", "Message" ) ); im.registerInformation( messageTable ); - informationGroupMessage.setRefreshFunction( this::update ); - //TODO: rmv button InformationGroup informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupPub ); InformationAction msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; - client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); + try { + client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); + } catch ( IllegalArgumentException e ) { + throw new RuntimeException( e ); + } return end; } ).withParameters( "topic", "msg" ); im.registerInformation( msgButton ); @@ -746,7 +874,7 @@ public MonitoringPage() { InformationGroup informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); im.addGroup( informationGroupReconn ); reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { - String end = "Reconnected to broker"; + String end = "Reconnecting to broker"; if ( client.getState().toString().equals( "DISCONNECTED" ) ) { run(); update(); @@ -770,6 +898,25 @@ public MonitoringPage() { } ); im.registerInformation( reconnButton ); + + // Test button + InformationGroup informationGroupTest = new InformationGroup( informationPage, "Test implementation with message: '10', topic:'button' and filterQuery:'value:10'." ).setOrder( 6 ); + im.addGroup( informationGroupTest ); + InformationAction testButton = new InformationAction( informationGroupTest, "Call MqttProcessor.processStream", ( parameters ) -> { + Transaction transaction = getTransaction(); + MqttMessage mqttMessage = new MqttMessage( "10", "button" ); + addMessageToQueue( "button", "10" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, "value:10", transaction.createStatement() ); + if ( streamProcessor.processStream() ) { + log.info( "Test returned true!" ); + } else { + log.info( "Test returned false." ); + } + return ""; + } ); + im.registerInformation( testButton ); + + } @@ -790,7 +937,6 @@ public void update() { } */ - topicsTable.reset(); if ( topicsMap.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index 887aa5140d..f2a3749883 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -16,48 +16,351 @@ package org.polypheny.db.mqtt; -import java.nio.charset.Charset; +import com.google.common.collect.ImmutableList; +import com.sun.jdi.BooleanType; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.polypheny.db.mqtt.ReceivedMqttMessage; +import org.bson.BsonArray; +import org.bson.BsonBoolean; +import org.bson.BsonDocument; +import org.bson.BsonInt32; +import org.bson.BsonString; +import org.bson.BsonValue; +import org.bson.conversions.Bson; +import org.polypheny.db.PolyImplementation; +import org.polypheny.db.algebra.AlgNode; +import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.algebra.logical.document.LogicalDocumentFilter; +import org.polypheny.db.algebra.logical.document.LogicalDocumentScan; +import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; +import org.polypheny.db.algebra.logical.relational.LogicalValues; +import org.polypheny.db.algebra.operators.OperatorName; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; +import org.polypheny.db.algebra.type.AlgDataTypeField; +import org.polypheny.db.algebra.type.AlgDataTypeFieldImpl; +import org.polypheny.db.algebra.type.AlgDataTypeImpl; +import org.polypheny.db.algebra.type.AlgRecordType; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.languages.NodeParseException; +import org.polypheny.db.languages.OperatorRegistry; +import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.languages.QueryParameters; +import org.polypheny.db.languages.mql.Mql.Type; +import org.polypheny.db.languages.mql.MqlAggregate; +import org.polypheny.db.languages.mql.MqlCount; +import org.polypheny.db.languages.mql.MqlDelete; +import org.polypheny.db.languages.mql.MqlFind; +import org.polypheny.db.languages.mql.MqlInsert; +import org.polypheny.db.languages.mql.MqlNode; +import org.polypheny.db.languages.mql.MqlQueryParameters; +import org.polypheny.db.languages.mql.MqlUpdate; +import org.polypheny.db.languages.mql.parser.MqlParser; +import org.polypheny.db.languages.mql.parser.MqlParser.MqlParserConfig; +import org.polypheny.db.languages.mql2alg.MqlToAlgConverter; +import org.polypheny.db.nodes.Node; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgTraitSet; +import org.polypheny.db.prepare.Context; +import org.polypheny.db.prepare.PolyphenyDbCatalogReader; +import org.polypheny.db.processing.ExtendedQueryParameters; +import org.polypheny.db.processing.Processor; +import org.polypheny.db.rex.RexBuilder; +import org.polypheny.db.rex.RexCall; +import org.polypheny.db.rex.RexInputRef; +import org.polypheny.db.rex.RexLiteral; +import org.polypheny.db.rex.RexNode; +import org.polypheny.db.schema.ModelTrait; import org.polypheny.db.stream.StreamProcessor; +import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.util.Pair; +import org.polypheny.db.util.SourceStringReader; @Slf4j public class MqttStreamProcessor implements StreamProcessor { - private final ReceivedMqttMessage receivedMqttMessage; + private final MqttMessage mqttMessage; + private final String filterQuery; private final StreamProcessor streamProcessor; - public MqttStreamProcessor(ReceivedMqttMessage receivedMqttMessage, Statement statement ) { - this.receivedMqttMessage = receivedMqttMessage; - this.streamProcessor = statement.getStreamProcessor( receivedMqttMessage.getMessage() ); + private Statement statement; + + public MqttStreamProcessor( MqttMessage mqttMessage,String filterQuery, Statement statement ) { + this.mqttMessage = mqttMessage; + this.filterQuery = filterQuery; + this.streamProcessor = statement.getStreamProcessor( mqttMessage.getMessage() ); + this.statement = statement; } - //TODO: receive all additional info from Wrapper around MqttStream - public String processStream( ) { - String info = extractInfo( this.receivedMqttMessage.getMessage(), this.receivedMqttMessage.getTopic() ); - if ( validateMsg( info ) ) { - log.info( "Extracted and validated message: {}", this.receivedMqttMessage.getMessage()); - return info; + public boolean processStream() { + AlgRoot root = processMqlQuery(); //buildFilter(); TODO + List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + log.info( res.toString() ); + return !res.isEmpty(); + + } + + + private AlgRoot processMqlQuery() { + AlgBuilder algBuilder = AlgBuilder.create( this.statement ); + + Processor mqlProcessor = statement.getTransaction().getProcessor( QueryLanguage.from( "mongo" ) ); + PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); + final AlgOptCluster cluster = AlgOptCluster.createDocument( statement.getQueryProcessor().getPlanner(), algBuilder.getRexBuilder() ); + + MqlToAlgConverter mqlConverter = new MqlToAlgConverter( mqlProcessor, catalogReader, cluster ); + + // QueryParameters parameters = new MqlQueryParameters( this.filterQuery, Catalog.getInstance().getDatabase( Catalog.defaultDatabaseId ).name,NamespaceType.DOCUMENT ); + + MqlFind find = (MqlFind) mqlProcessor.parse("db.collection.find({" + this.filterQuery + "})").get( 0 ); + + final AlgDataType rowType = + cluster.getTypeFactory() + .builder() + .add( "value", null, PolyType.INTEGER ) + .nullable( false ) + .build(); + final ImmutableList> tuples = + ImmutableList.of( + ImmutableList.of( (RexLiteral) + cluster.getRexBuilder().makeLiteral( + Integer.parseInt( this.mqttMessage.getMessage() ), + rowType.getFieldList().get( 0 ).getType(), + true) ) ); + +/* + RexCall queryValue = new RexCall( + cluster.getTypeFactory().createPolyType( PolyType.ANY ), + OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_QUERY_VALUE ), + Arrays.asList( + RexInputRef.of( 0, rowType ), + filter ) ); + */ + + AlgTraitSet docTraitSet = cluster.traitSet().replace( ModelTrait.DOCUMENT ); + LogicalValues msgValue = new LogicalValues( cluster, docTraitSet, rowType, tuples ); + //AlgNode input = LogicalDocumentValues.create( msgValue ); + + + BsonDocument msgDoc = new BsonDocument("value", new BsonInt32( Integer.parseInt( this.mqttMessage.getMessage() ) ) ); + List fields = new ArrayList<>(); + fields.add( new AlgDataTypeFieldImpl( "d", 0, algBuilder.getTypeFactory().createPolyType( PolyType.DOCUMENT ) ) ); + AlgDataType defaultRowType = new AlgRecordType( fields ); + + AlgNode input = new LogicalDocumentValues( cluster, defaultRowType, docTraitSet,ImmutableList.of( msgDoc ) ); + //AlgNode input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); + AlgRoot root; + root = mqlConverter.convert( find, input ); + + + + // insert message in alg tree + if ( this.mqttMessage.getMessage().contains( "[" ) ) { + //msg is array -> save with MQL_ITEM + //TODO: implekmt + /* + String[] msgArray = this.mqttMessage.getMessage().replace( "[", "" ).replace( "]", "" ).split( "," ); + BsonArray bsonValues = new BsonArray(msgArray.length); + for ( int i = 0; i < msgArray.length; i++ ) { + bsonValues.add( new BsonString( msgArray[i].trim() ) ); + } + List nodes = convertArray( key, bsonValues, rowType ); + RexNode call = new RexCall( any, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_ITEM ), Arrays.asList( nodes.get( 0 ), nodes.get( 1 ) ) ); +*/ + +// l. 2198 MqltoALg + /* + private RexCall getStringArray( List elements ) { + List rexNodes = new ArrayList<>(); + int maxSize = 0; + for ( String name : elements ) { + rexNodes.add( convertLiteral( new BsonString( name ) ) ); + maxSize = Math.max( name.length(), maxSize ); + } + + AlgDataType type = cluster.getTypeFactory().createArrayType( + cluster.getTypeFactory().createPolyType( PolyType.CHAR, maxSize ), + rexNodes.size() ); + return getArray( rexNodes, type ); + +*/ + root = null; + //--------------------------------------------------------------------- + + + } else if ( this.mqttMessage.getMessage().contains( "{" ) ) { + // msg in JSON format + //TODO: implekmt + /* + RexLiteral key = rexBuilder.makeLiteral( find.get ); + if ( values.get( "" ).isDocument() ) { + comparisionValue = values.get( "" ).asDocument(); + } + call = rexBuilder.makeCall( OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EXISTS, ); + + */ + root = null; + // --------------------------------------------------------------------- + + + } else { - log.error( "Message is not valid!" ); - return null; + //TODO: if msg is only one value: then field/key is empty String + // Abfrage ob value zum key Document ist weil dann gibt es einen anderen Operator als equals. + // only one value: + + // build Document out of message value: + // TODO: at end: change type of message to Object, so that also ints are supported + + + + if ( find.getQuery().get( "value" ).isDocument() ) { + String parentkey = ""; + // TODO: key is then: parentkey.key + /* + // other op than eq: + BsonDocument doc = values.get( "" ).asDocument(); + if ( doc.getFirstKey().equals( "" ) ) { + + } else if ( doc.getFirstKey().equals( "" ) ) { + + } + AlgNode node = LogicalDocumentValues.create( algBuilder.getCluster(), ImmutableList.of( msgDoc ) ); + */ + + root = null; + } else { + // op is equals + /* + RexNode queryLiteral = algBuilder.literal( find.getQuery().get( "value" ).asInt32().getValue() ); + RexInputRef msgRef = cluster.getRexBuilder().makeInputRef( input,0 ); + // build condition for where clause (=condition for saving) + AlgDataType type = cluster.getTypeFactory().createPolyType( PolyType.BOOLEAN ); + String[] fieldName ={"value"}; + AlgNode filter = algBuilder + //.values( fieldName, Integer.parseInt( this.mqttMessage.getMessage() ) ) + //algBuilder.call( OperatorRegistry.get( OperatorName.EQUALS ), algBuilder.field( "value" ), queryLiteral ) + .filter( new RexCall( type, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EQUALS ), Arrays.asList( msgRef, queryLiteral ) ) ) + .build(); + root = AlgRoot.of( filter, Kind.FILTER ); +*/ + } + } + + + // MqlToAlgConverter 1467 + /* + if ( key.equals( "$exists" ) ) { + return convertExists( bsonValue, key, rowType ); + } + + + // TODO: check if attibute exists + + */ + return root; } - private static boolean validateMsg( String msg ) { - //TODO: Implement - return true; + private RexNode translateQuery(BsonDocument bsonDocument, AlgDataType rowType, String parentKey, AlgBuilder algBuilder) { + // TODO: for-loop for more than one predicates + // parentkey = null + List nodes = new ArrayList<>(); + nodes.add( attachRef( bsonDocument.getFirstKey(), rowType ) ); + + if ( bsonDocument.get( bsonDocument.getFirstKey() ).isArray() ) { + /* TODO + List arr = convertArray( bsonDocument.getFirstKey(), bsonDocument.get( bsonDocument.getFirstKey() ).asArray(), true, rowType, "" ); + nodes.add( getArray(arr, algBuilder.getCluster().getTypeFactory().createArrayType( nullableAny, arr.size() ) ) ); + */ + //List arr = bsonValue.asArray().stream().map( this::convertLiteral ).collect( Collectors.toList() ); + //queryLiteral = getArray( arr, any ); + + } else { + AlgDataType type = algBuilder.getCluster().getTypeFactory().createPolyType( getPolyType( bsonDocument.get( bsonDocument.getFirstKey() ) ) ); + RexLiteral queryLiteral; + Pair valuePair = RexLiteral.convertType( getComparable( bsonDocument.get( bsonDocument.getFirstKey() ) ), type ); + queryLiteral = new RexLiteral( valuePair.left, type, valuePair.right ); + nodes.add( queryLiteral ); + } + AlgDataType type = algBuilder.getTypeFactory().createTypeWithNullability( algBuilder.getTypeFactory().createPolyType( PolyType.BOOLEAN ), true ); + RexNode predicate = new RexCall( type, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EQUALS ), nodes ); + return predicate; + + //return getFixedCall( operands, OperatorRegistry.get( OperatorName.AND ), PolyType.BOOLEAN ); } - private static String extractInfo( String msg, String topic ) { - //TODO: extract the needed Info only -> based on topic attribut on right side!!}; - return msg; + private Comparable getComparable( BsonValue value ) { + switch ( value.getBsonType() ) { + case INT32: + return value.asInt32().getValue(); + case DOUBLE: + return value.asDouble().getValue(); + case STRING: + return value.asString().getValue(); + } + throw new RuntimeException( "Not implemented Comparable transform" ); + } + + + private PolyType getPolyType( BsonValue bsonValue ) { + switch ( bsonValue.getBsonType() ) { + case DOUBLE: + return PolyType.DOUBLE; + case STRING: + return PolyType.CHAR; + case DOCUMENT: + return PolyType.JSON; + case ARRAY: + break; + case BOOLEAN: + return PolyType.BOOLEAN; + case INT32: + return PolyType.INTEGER; + } + throw new RuntimeException( "PolyType not implemented " ); + } + +/* + private AlgNode buildDocumentFilter( BsonDocument filterDoc, AlgNode msgNode, AlgDataType rowType ) { + + ArrayList operands = new ArrayList<>(); + + for ( Entry entry : filterDoc.entrySet() ) { + if ( entry.getKey().equals( "$regex" ) ) { + operands.add( convertRegex( filterDoc, parentKey, rowType ) ); + } else if ( !entry.getKey().equals( "$options" ) ) { + // normal handling + operands.add( convertEntry( entry.getKey(), parentKey, entry.getValue(), rowType ) ); + } + } + RexNode condition = getFixedCall( operands, OperatorRegistry.get( OperatorName.AND ), PolyType.BOOLEAN ); + + return LogicalDocumentFilter.create( msgNode, condition ); + } + + */ + + + private RexNode attachRef( String key, AlgDataType rowType ) { + AlgDataTypeField field = rowType.getField( key, false, false ); + return RexInputRef.of( field.getIndex(), rowType ); } @@ -66,4 +369,183 @@ public String getStream() { return streamProcessor.getStream(); } + + /** + * {@inheritDoc} + */ + @Override + public boolean validateContent( String msg ) { + //TODO: Implement + return true; + } + + + private String extractValue( String attributeName ) { + attributeName = attributeName; + int attributeStartIndex = this.mqttMessage.getMessage().indexOf( attributeName ); + if ( attributeStartIndex == -1 ) { + throw new RuntimeException( "The specified attribute could not be found in the received message!" ); + } + int attributeEndIndex = attributeStartIndex + attributeName.length(); + int valueStartIndex = this.mqttMessage.getMessage().indexOf( ":", attributeEndIndex ); + int valueEndIndex = this.mqttMessage.getMessage().indexOf( ",", attributeEndIndex ); + //TODO: Problem, wenn attribute als letztes ist, weil danach kein komma kommt sondern }. + //TODO: kann probleme geben, wenn comparission value mit " ist und attributeValue ohne " + String attributeValue = this.mqttMessage.getMessage().substring( valueStartIndex, valueEndIndex ).trim().replaceAll( "\"", "" ); + return attributeValue; + } + + + public AlgRoot buildFilter() { + AlgBuilder algBuilder = AlgBuilder.create( this.statement ); + //TODO: change if this is doch needed: + List predicatesList = buildPredicateRexNodes(this.filterQuery, algBuilder); + AlgNode algNode = algBuilder.filter( predicatesList ).build(); + return AlgRoot.of( algNode, Kind.FILTER ); + + +/* + String[] fieldname = {this.receivedMqttMessage.getTopic()}; + // --------- simple query ----------- + // extract operator for query + Operator queryOperator; + if ( query != null ) { + String[] queryArray = query.split( " " ); + switch ( queryArray[0].trim() ) { + case "<": + queryOperator = OperatorRegistry.get( OperatorName.LESS_THAN ); + break; + case "<=": + queryOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); + break; + case ">": + queryOperator = OperatorRegistry.get( OperatorName.GREATER_THAN ); + break; + case ">=": + queryOperator = OperatorRegistry.get( OperatorName.GREATER_THAN_OR_EQUAL ); + break; + case "=": + queryOperator = OperatorRegistry.get( OperatorName.EQUALS ); + break; + case "<>": + queryOperator = OperatorRegistry.get( OperatorName.NOT_EQUALS ); + break; + default: + throw new RuntimeException( "The operator in the filter query could not be identified!" ); + } + // extract other literal from query + RexNode queryLiteral = algBuilder.literal( queryArray[1].trim() ); + + // build condition for where clause (=condition for saving) + =algBuilder + .values( fieldname, this.receivedMqttMessage.getMessage() ) + .filter( algBuilder.call( queryOperator, algBuilder.field( this.receivedMqttMessage.getTopic() ), queryLiteral ) ) + .build(); + + } + + */ + } + + + private List buildPredicateRexNodes(String predicatesArray, AlgBuilder algBuilder) { + + List predicateList = new ArrayList<>(); + /* + // means query has a simple form: + + // complex form means query has attribute names in it: + for ( int i = 0; i < predicatesArray.length; i++ ) { + int operatorIndex; + int lengthOfOperator; + Operator predicateOperator; + if ( predicatesArray[i].contains( "<" ) ) { + operatorIndex = predicatesArray[i].indexOf( "<" ); + lengthOfOperator = 1; + predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN ); + } else if ( predicatesArray[i].contains( "<=" ) ) { + operatorIndex = predicatesArray[i].indexOf( "<=" ); + lengthOfOperator = 2; + predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); + } else if ( predicatesArray[i].contains( ">" ) ) { + operatorIndex = predicatesArray[i].indexOf( ">" ); + lengthOfOperator = 1; + predicateOperator = OperatorRegistry.get( OperatorName.GREATER_THAN ); + } else if ( predicatesArray[i].contains( ">=" ) ) { + operatorIndex = predicatesArray[i].indexOf( ">=" ); + lengthOfOperator = 2; + predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); + } else if ( predicatesArray[i].contains( "=" ) ) { + operatorIndex = predicatesArray[i].indexOf( "=" ); + lengthOfOperator = 1; + predicateOperator = OperatorRegistry.get( OperatorName.EQUALS ); + } else if ( predicatesArray[i].contains( "<>" ) ) { + operatorIndex = predicatesArray[i].indexOf( "<>" ); + lengthOfOperator = 2; + predicateOperator = OperatorRegistry.get( OperatorName.NOT_EQUALS ); + } else { + throw new RuntimeException( "MqttStreamProcessor could not recognize an operator." ); + } + RexNode comparisionLiteral = algBuilder.literal( predicatesArray[i].substring( operatorIndex + lengthOfOperator ).trim() ); + String[] attributeName = new String[1]; + String attributeValue; + if ( predicatesArray.length == 1 ) { + attributeName[0] = this.mqttMessage.getTopic(); + attributeValue = this.mqttMessage.getMessage(); + } else { + attributeName[0] = predicatesArray[i].substring( 0, operatorIndex ).trim(); + attributeValue = extractValue( attributeName[0] ); + if ( !validateContent( attributeValue ) ) { + throw new RuntimeException( "Content value was not valid." ); + } + } + algBuilder.values( attributeName, attributeValue ); + RexNode predicateNode = algBuilder.call(predicateOperator, algBuilder.field( attributeName[0] ), comparisionLiteral ); + predicateList.add( predicateNode ); + } + + */ + return predicateList; + } + + + List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { + + try { + // Prepare + PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); + log.debug( "AlgRoot was prepared." ); + + List> rows = result.getRows( statement, -1 ); + statement.getTransaction().commit(); + return rows; + } catch ( Throwable e ) { + log.error( "Error during execution of stream processor query", e ); + try { + statement.getTransaction().rollback(); + } catch ( TransactionException transactionException ) { + log.error( "Could not rollback", e ); + } + return null; + } + } + + + /** + * saves predicatesString in array seperated by commas + * @param predicatesString + * @return + */ + private String[] saveInArray( String predicatesString ) { + if ( predicatesString != null ) { + String[] queryArray = predicatesString.split( "," ); + for ( int i = 0; i < queryArray.length; i++ ) { + queryArray[i] = queryArray[i].trim(); + } + return queryArray; + } else { + return new String[0]; + } + } + } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java index 1cd8ea6452..5cbd46ebe9 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java @@ -16,10 +16,11 @@ package org.polypheny.db.mqtt; +import javax.management.Query; import lombok.Getter; import lombok.Setter; import org.polypheny.db.catalog.Catalog.NamespaceType; - +//TODO: javadoc public class ReceivedMqttMessage { private final MqttMessage msg; @@ -35,14 +36,19 @@ public class ReceivedMqttMessage { private final long databaseId; @Getter private final int userId; + /** + * if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is name of the topic or (if the subscribed topic + * has wildcards) the wildcardTopic + * if MqttStreamServer.collectionPerTopic = FALSE, then collectionName is the name of the common collection + */ @Getter - private final String collectionName; // if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is name of the topic - // if MqttStreamServer.collectionPerTopic = FALSE, then collectionName is the name of the common collection + private final String collectionName; public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, String uniqueNameOfInterface, long databaseId, int userId, String collectionName ) { this.msg = msg; this.namespaceName = namespaceName; this.namespaceType = namespaceType; + //TODO: schauen, wo namespaceId gebraucht wird. this.namespaceId = namespaceId; this.uniqueNameOfInterface = uniqueNameOfInterface; this.databaseId = databaseId; From 32dc64ec8125bc8f2885ac1207be8ce571aee1e3 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 3 Sep 2023 21:10:25 +0200 Subject: [PATCH 076/114] minor changes --- .../java/org/polypheny/db/mqtt/StreamCapture.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 341ab597dd..45e0c2f5e2 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -25,13 +25,6 @@ import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; -import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.NamespaceType; -import org.polypheny.db.catalog.Catalog.Pattern; -import org.polypheny.db.catalog.entity.CatalogCollection; -import org.polypheny.db.catalog.entity.CatalogSchema; -import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; -import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.prepare.Context; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; @@ -52,7 +45,7 @@ public class StreamCapture { } - public void handleContent( ReceivedMqttMessage receivedMqttMessage ) { + public void insert( ReceivedMqttMessage receivedMqttMessage ) { this.receivedMqttMessage = receivedMqttMessage; insertDocument( this.receivedMqttMessage.getCollectionName() ); } @@ -66,7 +59,6 @@ public void insertDocument( String collectionName) { // Builder which allows to construct the algebra tree which is equivalent to query and is executed AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); - // we insert document { age: 28, name: "David" } into the collection users BsonDocument document = new BsonDocument(); document.put( "source", new BsonString( this.receivedMqttMessage.getUniqueNameOfInterface() ) ); document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); @@ -74,7 +66,6 @@ public void insertDocument( String collectionName) { AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); - // we can then wrap the tree in an AlgRoot and execute it AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); // for inserts and all DML queries only a number is returned List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); From db647fedd6059b2caef09dfc43eb8a73827e2e2f Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 4 Sep 2023 13:45:42 +0200 Subject: [PATCH 077/114] fixed test and filtering --- .../enumerable/EnumerableDocumentValues.java | 82 +++++++++++++++++++ .../rules/DocumentToEnumerableRule.java | 17 ++++ .../db/plan/volcano/VolcanoPlanner.java | 1 + plugins/mqtt-stream/build.gradle | 1 + plugins/mqtt-stream/gradle.properties | 2 +- .../db/mqtt/MqttStreamProcessor.java | 35 +------- .../java/org/polypheny/db/mqtt/MqttTest.java | 2 +- 7 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableDocumentValues.java diff --git a/core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableDocumentValues.java b/core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableDocumentValues.java new file mode 100644 index 0000000000..7bfd0e2a14 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableDocumentValues.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.enumerable; + +import com.google.common.collect.ImmutableList; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Primitive; +import org.bson.BsonValue; +import org.polypheny.db.adapter.java.JavaTypeFactory; +import org.polypheny.db.algebra.core.document.DocumentValues; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeField; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgTraitSet; +import org.polypheny.db.schema.ModelTrait; +import org.polypheny.db.util.BuiltInMethod; + +public class EnumerableDocumentValues extends DocumentValues implements EnumerableAlg { + + /** + * Creates a {@link DocumentValues}. + * {@link ModelTrait#DOCUMENT} node, which contains values. + * + * @param cluster + * @param traitSet + * @param rowType + * @param documentTuples + */ + public EnumerableDocumentValues( AlgOptCluster cluster, AlgTraitSet traitSet, AlgDataType rowType, List documentTuples ) { + super( cluster, traitSet, rowType, ImmutableList.copyOf( documentTuples ) ); + } + + + @Override + public Result implement( EnumerableAlgImplementor implementor, Prefer pref ) { + final JavaTypeFactory typeFactory = (JavaTypeFactory) getCluster().getTypeFactory(); + final BlockBuilder builder = new BlockBuilder(); + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), + getRowType(), + pref.preferCustom() ); + final Type rowClass = physType.getJavaRowType(); + + final List expressions = new ArrayList<>(); + final List fields = rowType.getFieldList(); + for ( BsonValue doc : documentTuples ) { + final List literals = new ArrayList<>(); + + literals.add( Expressions.constant( doc.asDocument().toJson() ) ); + + expressions.add( physType.record( literals ) ); + } + builder.add( + Expressions.return_( + null, + Expressions.call( + BuiltInMethod.AS_ENUMERABLE.method, + Expressions.newArrayInit( Primitive.box( rowClass ), expressions ) ) ) ); + return implementor.result( physType, builder.toBlock() ); + } + +} diff --git a/core/src/main/java/org/polypheny/db/algebra/rules/DocumentToEnumerableRule.java b/core/src/main/java/org/polypheny/db/algebra/rules/DocumentToEnumerableRule.java index 0530c25fff..d1570b9615 100644 --- a/core/src/main/java/org/polypheny/db/algebra/rules/DocumentToEnumerableRule.java +++ b/core/src/main/java/org/polypheny/db/algebra/rules/DocumentToEnumerableRule.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.enumerable.EnumerableAggregate; import org.polypheny.db.adapter.enumerable.EnumerableConvention; +import org.polypheny.db.adapter.enumerable.EnumerableDocumentValues; import org.polypheny.db.adapter.enumerable.EnumerableFilter; import org.polypheny.db.adapter.enumerable.EnumerableLimit; import org.polypheny.db.adapter.enumerable.EnumerableProject; @@ -30,6 +31,7 @@ import org.polypheny.db.algebra.logical.document.LogicalDocumentFilter; import org.polypheny.db.algebra.logical.document.LogicalDocumentProject; import org.polypheny.db.algebra.logical.document.LogicalDocumentSort; +import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; import org.polypheny.db.plan.AlgOptRule; import org.polypheny.db.plan.AlgOptRuleCall; import org.polypheny.db.plan.AlgOptRuleOperand; @@ -43,6 +45,8 @@ public class DocumentToEnumerableRule extends AlgOptRule { public static DocumentToEnumerableRule FILTER_TO_ENUMERABLE = new DocumentToEnumerableRule( Type.FILTER, operand( LogicalDocumentFilter.class, any() ), "DOCUMENT_FILTER_TO_ENUMERABLE" ); public static DocumentToEnumerableRule SORT_TO_ENUMERABLE = new DocumentToEnumerableRule( Type.SORT, operand( LogicalDocumentSort.class, any() ), "DOCUMENT_SORT_TO_ENUMERABLE" ); + public static DocumentToEnumerableRule VALUES_TO_ENUMERABLE = new DocumentToEnumerableRule( Type.VALUES, operand( LogicalDocumentValues.class, any() ), "DOCUMENT_VALUES_TO_ENUMERABLE" ); + private final Type type; @@ -62,6 +66,8 @@ public void onMatch( AlgOptRuleCall call ) { convertAggregate( call ); } else if ( type == Type.SORT ) { convertSort( call ); + } else if ( type == Type.VALUES ) { + convertValues( call ); } else { throw new UnsupportedOperationException( "This document is not supported." ); } @@ -69,6 +75,17 @@ public void onMatch( AlgOptRuleCall call ) { } + private void convertValues( AlgOptRuleCall call ) { + LogicalDocumentValues values = call.alg( 0 ); + AlgTraitSet out = values.getTraitSet().replace( EnumerableConvention.INSTANCE ); + + EnumerableDocumentValues enumerable = new EnumerableDocumentValues( values.getCluster(), out, values.getRowType(), values.documentTuples ); + call.transformTo( enumerable ); + + // call.transformTo( values.getRelationalEquivalent() ); + } + + private void convertSort( AlgOptRuleCall call ) { LogicalDocumentSort sort = call.alg( 0 ); AlgTraitSet out = sort.getTraitSet().replace( EnumerableConvention.INSTANCE ); diff --git a/core/src/main/java/org/polypheny/db/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/polypheny/db/plan/volcano/VolcanoPlanner.java index e6abfe2f19..f831335452 100644 --- a/core/src/main/java/org/polypheny/db/plan/volcano/VolcanoPlanner.java +++ b/core/src/main/java/org/polypheny/db/plan/volcano/VolcanoPlanner.java @@ -830,6 +830,7 @@ public void registerModelRules() { addRule( DocumentToEnumerableRule.FILTER_TO_ENUMERABLE ); addRule( DocumentToEnumerableRule.AGGREGATE_TO_ENUMERABLE ); addRule( DocumentToEnumerableRule.SORT_TO_ENUMERABLE ); + addRule( DocumentToEnumerableRule.VALUES_TO_ENUMERABLE ); // Relational } diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 80cdc02fc8..b02282606a 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -22,6 +22,7 @@ dependencies { testImplementation project(path: ":dbms") testImplementation project(path: ":core", configuration: "tests") testImplementation project(path: ":core") + testImplementation project(":plugins:mql-language") testCompileOnly group: 'org.pf4j', name: 'pf4j', version: pf4jVersion testImplementation group: "junit", name: "junit", version: junit_version diff --git a/plugins/mqtt-stream/gradle.properties b/plugins/mqtt-stream/gradle.properties index 82bbbbe18d..888b2d1806 100644 --- a/plugins/mqtt-stream/gradle.properties +++ b/plugins/mqtt-stream/gradle.properties @@ -19,7 +19,7 @@ pluginVersion = 0.0.1 pluginId = mqtt-stream pluginClass = org.polypheny.db.mqtt.MqttStreamPlugin pluginProvider = The Polypheny Project -pluginDependencies = +pluginDependencies = mql-language pluginUrlPath = pluginCategories = interface pluginPolyDependencies = diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index f2a3749883..c02826b38e 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -17,63 +17,32 @@ package org.polypheny.db.mqtt; import com.google.common.collect.ImmutableList; -import com.sun.jdi.BooleanType; -import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.bson.BsonArray; -import org.bson.BsonBoolean; import org.bson.BsonDocument; import org.bson.BsonInt32; -import org.bson.BsonString; import org.bson.BsonValue; -import org.bson.conversions.Bson; import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; -import org.polypheny.db.algebra.logical.document.LogicalDocumentFilter; -import org.polypheny.db.algebra.logical.document.LogicalDocumentScan; import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; import org.polypheny.db.algebra.logical.relational.LogicalValues; import org.polypheny.db.algebra.operators.OperatorName; import org.polypheny.db.algebra.type.AlgDataType; -import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.algebra.type.AlgDataTypeFieldImpl; -import org.polypheny.db.algebra.type.AlgDataTypeImpl; import org.polypheny.db.algebra.type.AlgRecordType; -import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.NamespaceType; -import org.polypheny.db.languages.NodeParseException; import org.polypheny.db.languages.OperatorRegistry; import org.polypheny.db.languages.QueryLanguage; -import org.polypheny.db.languages.QueryParameters; -import org.polypheny.db.languages.mql.Mql.Type; -import org.polypheny.db.languages.mql.MqlAggregate; -import org.polypheny.db.languages.mql.MqlCount; -import org.polypheny.db.languages.mql.MqlDelete; import org.polypheny.db.languages.mql.MqlFind; -import org.polypheny.db.languages.mql.MqlInsert; -import org.polypheny.db.languages.mql.MqlNode; -import org.polypheny.db.languages.mql.MqlQueryParameters; -import org.polypheny.db.languages.mql.MqlUpdate; -import org.polypheny.db.languages.mql.parser.MqlParser; -import org.polypheny.db.languages.mql.parser.MqlParser.MqlParserConfig; import org.polypheny.db.languages.mql2alg.MqlToAlgConverter; -import org.polypheny.db.nodes.Node; import org.polypheny.db.plan.AlgOptCluster; import org.polypheny.db.plan.AlgTraitSet; import org.polypheny.db.prepare.Context; import org.polypheny.db.prepare.PolyphenyDbCatalogReader; -import org.polypheny.db.processing.ExtendedQueryParameters; import org.polypheny.db.processing.Processor; -import org.polypheny.db.rex.RexBuilder; import org.polypheny.db.rex.RexCall; import org.polypheny.db.rex.RexInputRef; import org.polypheny.db.rex.RexLiteral; @@ -82,11 +51,9 @@ import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; -import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.type.PolyType; import org.polypheny.db.util.Pair; -import org.polypheny.db.util.SourceStringReader; @Slf4j public class MqttStreamProcessor implements StreamProcessor { @@ -126,7 +93,7 @@ private AlgRoot processMqlQuery() { // QueryParameters parameters = new MqlQueryParameters( this.filterQuery, Catalog.getInstance().getDatabase( Catalog.defaultDatabaseId ).name,NamespaceType.DOCUMENT ); - MqlFind find = (MqlFind) mqlProcessor.parse("db.collection.find({" + this.filterQuery + "})").get( 0 ); + MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "collection", this.filterQuery ) ).get( 0 ); final AlgDataType rowType = cluster.getTypeFactory() diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java index 25348b006c..c398fad5e0 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java @@ -32,7 +32,7 @@ public static void init() { public void algTest() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); - String filterQuery = "db.buttontest.find({value:10})"; + String filterQuery = "{\"value\":11}"; MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); streamProcessor.processStream(); From 8b2c078b59bc32fa5758864b86b6b826eb61d2d5 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 5 Sep 2023 16:51:11 +0200 Subject: [PATCH 078/114] changed ModelTrait to Document --- .../db/algebra/logical/document/LogicalDocumentValues.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java b/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java index 524569295f..083503b78e 100644 --- a/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java +++ b/core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java @@ -78,7 +78,7 @@ public class LogicalDocumentValues extends DocumentValues implements RelationalT * @param tuples the documents in their native BSON format */ public LogicalDocumentValues( AlgOptCluster cluster, AlgDataType defaultRowType, AlgTraitSet traitSet, ImmutableList tuples ) { - super( cluster, traitSet, defaultRowType, tuples ); + super( cluster, traitSet.replace( ModelTrait.DOCUMENT ), defaultRowType, tuples ); } From e49b7257c87e364bf60b8324f3a9fc5e986535ee Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 5 Sep 2023 16:53:27 +0200 Subject: [PATCH 079/114] changed structure of convert method & added '$$ROOT' as operator --- .../languages/mql2alg/MqlToAlgConverter.java | 86 +--- .../db/mqtt/MqttStreamProcessor.java | 441 ++---------------- 2 files changed, 58 insertions(+), 469 deletions(-) diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java index 5fbc6f61b7..6e17253cc5 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java @@ -29,6 +29,7 @@ import java.util.Objects; import java.util.function.BiFunction; import java.util.stream.Collectors; +import javax.annotation.Nullable; import lombok.Getter; import org.bson.BsonArray; import org.bson.BsonBoolean; @@ -217,6 +218,7 @@ public class MqlToAlgConverter { operators.add( "$all" ); operators.add( "$elemMatch" ); operators.add( "$size" ); + operators.add( "$$ROOT" ); } @@ -260,7 +262,8 @@ public AlgRoot convert( Node query, QueryParameters parameters ) { this.parameters = (MqlQueryParameters) parameters; this.defaultDatabase = ((MqlQueryParameters) parameters).getDatabase(); if ( query instanceof MqlCollectionStatement ) { - return convert( (MqlCollectionStatement) query ); + AlgNode scanNode = createScan( (MqlCollectionStatement) query ); + return convert( (MqlCollectionStatement) query, scanNode ); } throw new RuntimeException( "DML or DQL need a collection" ); } @@ -270,73 +273,12 @@ public AlgRoot convert( Node query, QueryParameters parameters ) { * Converts the initial MongoQl by stepping through it iteratively * * @param query the query in MqlNode format - * @return the {@link AlgNode} format of the initial query - */ - public AlgRoot convert( MqlCollectionStatement query ) { - Type kind = query.getMqlKind(); - this.entity = getEntity( query, defaultDatabase ); - if ( entity == null ) { - throw new RuntimeException( "The used collection does not exist." ); - } - - AlgNode node; - - if ( entity.getTable().getSchemaType() == NamespaceType.RELATIONAL ) { - _dataExists = false; - } - - node = LogicalDocumentScan.create( cluster, entity ); - this.usesDocumentModel = true; - - AlgDataType rowType = entity.getRowType(); - - this.builder = new RexBuilder( cluster.getTypeFactory() ); - - AlgRoot root; - - switch ( kind ) { - case FIND: - AlgNode find = convertFind( (MqlFind) query, rowType, node ); - root = AlgRoot.of( find, find.getRowType(), Kind.SELECT ); - break; - case COUNT: - AlgNode count = convertCount( (MqlCount) query, rowType, node ); - root = AlgRoot.of( count, count.getRowType(), Kind.SELECT ); - break; - case AGGREGATE: - AlgNode aggregate = convertAggregate( (MqlAggregate) query, rowType, node ); - root = AlgRoot.of( aggregate, Kind.SELECT ); - break; - /// dmls - case INSERT: - root = AlgRoot.of( convertInsert( (MqlInsert) query, entity ), Kind.INSERT ); - break; - case DELETE: - case FIND_DELETE: - root = AlgRoot.of( convertDelete( (MqlDelete) query, entity, node ), Kind.DELETE ); - break; - case UPDATE: - root = AlgRoot.of( convertUpdate( (MqlUpdate) query, entity, node ), Kind.UPDATE ); - break; - default: - throw new IllegalStateException( "Unexpected value: " + kind ); - } - /*if ( usesDocumentModel ) { - root.usesDocumentModel = true; - }*/ - return root; - } - - - /** - * Converts the initial MongoQl by stepping through it iteratively - * but queries on a given input instead of a given collection - * @param query the query in MqlNode format - * @param input the value that should be queried + * @param input the input that should be queried * @return the {@link AlgNode} format of the initial query */ public AlgRoot convert( MqlCollectionStatement query, AlgNode input ) { Type kind = query.getMqlKind(); + this.usesDocumentModel = true; AlgDataType rowType = input.getRowType(); @@ -379,6 +321,20 @@ public AlgRoot convert( MqlCollectionStatement query, AlgNode input ) { } + private AlgNode createScan( MqlCollectionStatement query ) { + this.entity = getEntity( query, defaultDatabase ); + if ( entity == null ) { + throw new RuntimeException( "The used collection does not exist." ); + } + + if ( entity.getTable().getSchemaType() == NamespaceType.RELATIONAL ) { + _dataExists = false; + } + + return LogicalDocumentScan.create( cluster, entity ); + } + + private AlgOptTable getEntity( MqlCollectionStatement query, String dbSchemaName ) { List names = ImmutableList.of( dbSchemaName, query.getCollection() ); @@ -1530,6 +1486,8 @@ private RexNode convertEntry( String key, String parentKey, BsonValue bsonValue, return convertElemMatch( bsonValue, parentKey, rowType ); } else if ( key.equals( "$size" ) ) { return convertSize( bsonValue, parentKey, rowType ); + } else if ( key.equals( "$$ROOT" ) ) { + return convertField( parentKey == null ? key : parentKey + "." + key, bsonValue, rowType ); } return translateLogical( key, parentKey, bsonValue, rowType ); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index c02826b38e..dfa76e21cf 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -20,61 +20,43 @@ import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.bson.BsonArray; import org.bson.BsonDocument; import org.bson.BsonInt32; +import org.bson.BsonString; import org.bson.BsonValue; import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; -import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; -import org.polypheny.db.algebra.logical.relational.LogicalValues; -import org.polypheny.db.algebra.operators.OperatorName; -import org.polypheny.db.algebra.type.AlgDataType; -import org.polypheny.db.algebra.type.AlgDataTypeField; -import org.polypheny.db.algebra.type.AlgDataTypeFieldImpl; -import org.polypheny.db.algebra.type.AlgRecordType; -import org.polypheny.db.languages.OperatorRegistry; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.languages.mql.MqlFind; import org.polypheny.db.languages.mql2alg.MqlToAlgConverter; import org.polypheny.db.plan.AlgOptCluster; -import org.polypheny.db.plan.AlgTraitSet; import org.polypheny.db.prepare.Context; import org.polypheny.db.prepare.PolyphenyDbCatalogReader; import org.polypheny.db.processing.Processor; -import org.polypheny.db.rex.RexCall; -import org.polypheny.db.rex.RexInputRef; -import org.polypheny.db.rex.RexLiteral; -import org.polypheny.db.rex.RexNode; -import org.polypheny.db.schema.ModelTrait; import org.polypheny.db.stream.StreamProcessor; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.TransactionException; -import org.polypheny.db.type.PolyType; -import org.polypheny.db.util.Pair; @Slf4j public class MqttStreamProcessor implements StreamProcessor { - - private final MqttMessage mqttMessage; private final String filterQuery; private final StreamProcessor streamProcessor; - - private Statement statement; + private final Statement statement; - public MqttStreamProcessor( MqttMessage mqttMessage,String filterQuery, Statement statement ) { - this.mqttMessage = mqttMessage; + public MqttStreamProcessor( MqttMessage mqttMessage, String filterQuery, Statement statement ) { this.filterQuery = filterQuery; this.streamProcessor = statement.getStreamProcessor( mqttMessage.getMessage() ); this.statement = statement; } - public boolean processStream() { - AlgRoot root = processMqlQuery(); //buildFilter(); TODO + public boolean applyFilter() { + AlgRoot root = processMqlQuery(); List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); log.info( res.toString() ); return !res.isEmpty(); @@ -84,250 +66,48 @@ public boolean processStream() { private AlgRoot processMqlQuery() { AlgBuilder algBuilder = AlgBuilder.create( this.statement ); - Processor mqlProcessor = statement.getTransaction().getProcessor( QueryLanguage.from( "mongo" ) ); PolyphenyDbCatalogReader catalogReader = statement.getTransaction().getCatalogReader(); final AlgOptCluster cluster = AlgOptCluster.createDocument( statement.getQueryProcessor().getPlanner(), algBuilder.getRexBuilder() ); - MqlToAlgConverter mqlConverter = new MqlToAlgConverter( mqlProcessor, catalogReader, cluster ); - // QueryParameters parameters = new MqlQueryParameters( this.filterQuery, Catalog.getInstance().getDatabase( Catalog.defaultDatabaseId ).name,NamespaceType.DOCUMENT ); - - MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "collection", this.filterQuery ) ).get( 0 ); - - final AlgDataType rowType = - cluster.getTypeFactory() - .builder() - .add( "value", null, PolyType.INTEGER ) - .nullable( false ) - .build(); - final ImmutableList> tuples = - ImmutableList.of( - ImmutableList.of( (RexLiteral) - cluster.getRexBuilder().makeLiteral( - Integer.parseInt( this.mqttMessage.getMessage() ), - rowType.getFieldList().get( 0 ).getType(), - true) ) ); - -/* - RexCall queryValue = new RexCall( - cluster.getTypeFactory().createPolyType( PolyType.ANY ), - OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_QUERY_VALUE ), - Arrays.asList( - RexInputRef.of( 0, rowType ), - filter ) ); - */ - - AlgTraitSet docTraitSet = cluster.traitSet().replace( ModelTrait.DOCUMENT ); - LogicalValues msgValue = new LogicalValues( cluster, docTraitSet, rowType, tuples ); - //AlgNode input = LogicalDocumentValues.create( msgValue ); - - - BsonDocument msgDoc = new BsonDocument("value", new BsonInt32( Integer.parseInt( this.mqttMessage.getMessage() ) ) ); - List fields = new ArrayList<>(); - fields.add( new AlgDataTypeFieldImpl( "d", 0, algBuilder.getTypeFactory().createPolyType( PolyType.DOCUMENT ) ) ); - AlgDataType defaultRowType = new AlgRecordType( fields ); - - AlgNode input = new LogicalDocumentValues( cluster, defaultRowType, docTraitSet,ImmutableList.of( msgDoc ) ); - //AlgNode input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); - AlgRoot root; - root = mqlConverter.convert( find, input ); - - - - // insert message in alg tree - if ( this.mqttMessage.getMessage().contains( "[" ) ) { - //msg is array -> save with MQL_ITEM - //TODO: implekmt - /* - String[] msgArray = this.mqttMessage.getMessage().replace( "[", "" ).replace( "]", "" ).split( "," ); - BsonArray bsonValues = new BsonArray(msgArray.length); - for ( int i = 0; i < msgArray.length; i++ ) { - bsonValues.add( new BsonString( msgArray[i].trim() ) ); - } - List nodes = convertArray( key, bsonValues, rowType ); - RexNode call = new RexCall( any, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_ITEM ), Arrays.asList( nodes.get( 0 ), nodes.get( 1 ) ) ); -*/ - -// l. 2198 MqltoALg - /* - private RexCall getStringArray( List elements ) { - List rexNodes = new ArrayList<>(); - int maxSize = 0; - for ( String name : elements ) { - rexNodes.add( convertLiteral( new BsonString( name ) ) ); - maxSize = Math.max( name.length(), maxSize ); - } - - AlgDataType type = cluster.getTypeFactory().createArrayType( - cluster.getTypeFactory().createPolyType( PolyType.CHAR, maxSize ), - rexNodes.size() ); - return getArray( rexNodes, type ); - -*/ - root = null; - //--------------------------------------------------------------------- - - - } else if ( this.mqttMessage.getMessage().contains( "{" ) ) { - // msg in JSON format - //TODO: implekmt - /* - RexLiteral key = rexBuilder.makeLiteral( find.get ); - if ( values.get( "" ).isDocument() ) { - comparisionValue = values.get( "" ).asDocument(); - } - call = rexBuilder.makeCall( OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EXISTS, ); - - */ - root = null; - // --------------------------------------------------------------------- - - - + MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "null", this.filterQuery ) ).get( 0 ); + String msg = getStream(); + AlgNode input; + if ( msg.contains( "{" ) && msg.contains( "}" ) ) { + // msg is in JSON format + BsonDocument msgDoc = BsonDocument.parse(msg); + input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); + } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { + // msg is an array + List values = arrayToBsonList(msg); + BsonDocument msgDoc = new BsonDocument( "$$ROOT", new BsonArray( values ) ); + input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); } else { - //TODO: if msg is only one value: then field/key is empty String - // Abfrage ob value zum key Document ist weil dann gibt es einen anderen Operator als equals. - // only one value: - - // build Document out of message value: - // TODO: at end: change type of message to Object, so that also ints are supported - - - - if ( find.getQuery().get( "value" ).isDocument() ) { - String parentkey = ""; - // TODO: key is then: parentkey.key - /* - // other op than eq: - BsonDocument doc = values.get( "" ).asDocument(); - if ( doc.getFirstKey().equals( "" ) ) { - - } else if ( doc.getFirstKey().equals( "" ) ) { - - } - AlgNode node = LogicalDocumentValues.create( algBuilder.getCluster(), ImmutableList.of( msgDoc ) ); - */ - - root = null; - } else { - // op is equals - /* - RexNode queryLiteral = algBuilder.literal( find.getQuery().get( "value" ).asInt32().getValue() ); - RexInputRef msgRef = cluster.getRexBuilder().makeInputRef( input,0 ); - // build condition for where clause (=condition for saving) - AlgDataType type = cluster.getTypeFactory().createPolyType( PolyType.BOOLEAN ); - String[] fieldName ={"value"}; - AlgNode filter = algBuilder - //.values( fieldName, Integer.parseInt( this.mqttMessage.getMessage() ) ) - //algBuilder.call( OperatorRegistry.get( OperatorName.EQUALS ), algBuilder.field( "value" ), queryLiteral ) - .filter( new RexCall( type, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EQUALS ), Arrays.asList( msgRef, queryLiteral ) ) ) - .build(); - root = AlgRoot.of( filter, Kind.FILTER ); -*/ - } - } - - - // MqlToAlgConverter 1467 - /* - if ( key.equals( "$exists" ) ) { - return convertExists( bsonValue, key, rowType ); - - } - - - // TODO: check if attibute exists - - */ - return root; - } - - - private RexNode translateQuery(BsonDocument bsonDocument, AlgDataType rowType, String parentKey, AlgBuilder algBuilder) { - // TODO: for-loop for more than one predicates - // parentkey = null - List nodes = new ArrayList<>(); - nodes.add( attachRef( bsonDocument.getFirstKey(), rowType ) ); - - if ( bsonDocument.get( bsonDocument.getFirstKey() ).isArray() ) { - /* TODO - List arr = convertArray( bsonDocument.getFirstKey(), bsonDocument.get( bsonDocument.getFirstKey() ).asArray(), true, rowType, "" ); - nodes.add( getArray(arr, algBuilder.getCluster().getTypeFactory().createArrayType( nullableAny, arr.size() ) ) ); - */ - //List arr = bsonValue.asArray().stream().map( this::convertLiteral ).collect( Collectors.toList() ); - //queryLiteral = getArray( arr, any ); - - } else { - AlgDataType type = algBuilder.getCluster().getTypeFactory().createPolyType( getPolyType( bsonDocument.get( bsonDocument.getFirstKey() ) ) ); - RexLiteral queryLiteral; - Pair valuePair = RexLiteral.convertType( getComparable( bsonDocument.get( bsonDocument.getFirstKey() ) ), type ); - queryLiteral = new RexLiteral( valuePair.left, type, valuePair.right ); - nodes.add( queryLiteral ); - } - AlgDataType type = algBuilder.getTypeFactory().createTypeWithNullability( algBuilder.getTypeFactory().createPolyType( PolyType.BOOLEAN ), true ); - RexNode predicate = new RexCall( type, OperatorRegistry.get( QueryLanguage.from( "mongo" ), OperatorName.MQL_EQUALS ), nodes ); - return predicate; - - //return getFixedCall( operands, OperatorRegistry.get( OperatorName.AND ), PolyType.BOOLEAN ); - } - - - private Comparable getComparable( BsonValue value ) { - switch ( value.getBsonType() ) { - case INT32: - return value.asInt32().getValue(); - case DOUBLE: - return value.asDouble().getValue(); - case STRING: - return value.asString().getValue(); + // msg is single value + BsonDocument msgDoc = new BsonDocument( "$$ROOT", new BsonInt32( Integer.parseInt( msg ) ) ); + input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); } - throw new RuntimeException( "Not implemented Comparable transform" ); - } - private PolyType getPolyType( BsonValue bsonValue ) { - switch ( bsonValue.getBsonType() ) { - case DOUBLE: - return PolyType.DOUBLE; - case STRING: - return PolyType.CHAR; - case DOCUMENT: - return PolyType.JSON; - case ARRAY: - break; - case BOOLEAN: - return PolyType.BOOLEAN; - case INT32: - return PolyType.INTEGER; - } - throw new RuntimeException( "PolyType not implemented " ); + return mqlConverter.convert( find, input ); } -/* - private AlgNode buildDocumentFilter( BsonDocument filterDoc, AlgNode msgNode, AlgDataType rowType ) { - - ArrayList operands = new ArrayList<>(); - for ( Entry entry : filterDoc.entrySet() ) { - if ( entry.getKey().equals( "$regex" ) ) { - operands.add( convertRegex( filterDoc, parentKey, rowType ) ); - } else if ( !entry.getKey().equals( "$options" ) ) { - // normal handling - operands.add( convertEntry( entry.getKey(), parentKey, entry.getValue(), rowType ) ); - } + /** + * converts the array, currently a String into a list of BSON values + * @param msg + */ + private List arrayToBsonList( String msg ) { + msg = msg.replace( "[", "" ).replace( "]", "" ); + String[] array = msg.split( "," ); + List list = new ArrayList<>(array.length); + for ( String stringValue : array ) { + //TODO: if BsonString dont work -> try with BsonInt + BsonString bsonValue = new BsonString( stringValue.trim() ); + list.add( bsonValue ); } - RexNode condition = getFixedCall( operands, OperatorRegistry.get( OperatorName.AND ), PolyType.BOOLEAN ); - - return LogicalDocumentFilter.create( msgNode, condition ); - } - - */ - - - private RexNode attachRef( String key, AlgDataType rowType ) { - AlgDataTypeField field = rowType.getField( key, false, false ); - return RexInputRef.of( field.getIndex(), rowType ); + return list; } @@ -347,142 +127,11 @@ public boolean validateContent( String msg ) { } - private String extractValue( String attributeName ) { - attributeName = attributeName; - int attributeStartIndex = this.mqttMessage.getMessage().indexOf( attributeName ); - if ( attributeStartIndex == -1 ) { - throw new RuntimeException( "The specified attribute could not be found in the received message!" ); - } - int attributeEndIndex = attributeStartIndex + attributeName.length(); - int valueStartIndex = this.mqttMessage.getMessage().indexOf( ":", attributeEndIndex ); - int valueEndIndex = this.mqttMessage.getMessage().indexOf( ",", attributeEndIndex ); - //TODO: Problem, wenn attribute als letztes ist, weil danach kein komma kommt sondern }. - //TODO: kann probleme geben, wenn comparission value mit " ist und attributeValue ohne " - String attributeValue = this.mqttMessage.getMessage().substring( valueStartIndex, valueEndIndex ).trim().replaceAll( "\"", "" ); - return attributeValue; - } - - - public AlgRoot buildFilter() { - AlgBuilder algBuilder = AlgBuilder.create( this.statement ); - //TODO: change if this is doch needed: - List predicatesList = buildPredicateRexNodes(this.filterQuery, algBuilder); - AlgNode algNode = algBuilder.filter( predicatesList ).build(); - return AlgRoot.of( algNode, Kind.FILTER ); - - -/* - String[] fieldname = {this.receivedMqttMessage.getTopic()}; - // --------- simple query ----------- - // extract operator for query - Operator queryOperator; - if ( query != null ) { - String[] queryArray = query.split( " " ); - switch ( queryArray[0].trim() ) { - case "<": - queryOperator = OperatorRegistry.get( OperatorName.LESS_THAN ); - break; - case "<=": - queryOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); - break; - case ">": - queryOperator = OperatorRegistry.get( OperatorName.GREATER_THAN ); - break; - case ">=": - queryOperator = OperatorRegistry.get( OperatorName.GREATER_THAN_OR_EQUAL ); - break; - case "=": - queryOperator = OperatorRegistry.get( OperatorName.EQUALS ); - break; - case "<>": - queryOperator = OperatorRegistry.get( OperatorName.NOT_EQUALS ); - break; - default: - throw new RuntimeException( "The operator in the filter query could not be identified!" ); - } - // extract other literal from query - RexNode queryLiteral = algBuilder.literal( queryArray[1].trim() ); - - // build condition for where clause (=condition for saving) - =algBuilder - .values( fieldname, this.receivedMqttMessage.getMessage() ) - .filter( algBuilder.call( queryOperator, algBuilder.field( this.receivedMqttMessage.getTopic() ), queryLiteral ) ) - .build(); - - } - - */ - } - - - private List buildPredicateRexNodes(String predicatesArray, AlgBuilder algBuilder) { - - List predicateList = new ArrayList<>(); - /* - // means query has a simple form: - - // complex form means query has attribute names in it: - for ( int i = 0; i < predicatesArray.length; i++ ) { - int operatorIndex; - int lengthOfOperator; - Operator predicateOperator; - if ( predicatesArray[i].contains( "<" ) ) { - operatorIndex = predicatesArray[i].indexOf( "<" ); - lengthOfOperator = 1; - predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN ); - } else if ( predicatesArray[i].contains( "<=" ) ) { - operatorIndex = predicatesArray[i].indexOf( "<=" ); - lengthOfOperator = 2; - predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); - } else if ( predicatesArray[i].contains( ">" ) ) { - operatorIndex = predicatesArray[i].indexOf( ">" ); - lengthOfOperator = 1; - predicateOperator = OperatorRegistry.get( OperatorName.GREATER_THAN ); - } else if ( predicatesArray[i].contains( ">=" ) ) { - operatorIndex = predicatesArray[i].indexOf( ">=" ); - lengthOfOperator = 2; - predicateOperator = OperatorRegistry.get( OperatorName.LESS_THAN_OR_EQUAL ); - } else if ( predicatesArray[i].contains( "=" ) ) { - operatorIndex = predicatesArray[i].indexOf( "=" ); - lengthOfOperator = 1; - predicateOperator = OperatorRegistry.get( OperatorName.EQUALS ); - } else if ( predicatesArray[i].contains( "<>" ) ) { - operatorIndex = predicatesArray[i].indexOf( "<>" ); - lengthOfOperator = 2; - predicateOperator = OperatorRegistry.get( OperatorName.NOT_EQUALS ); - } else { - throw new RuntimeException( "MqttStreamProcessor could not recognize an operator." ); - } - RexNode comparisionLiteral = algBuilder.literal( predicatesArray[i].substring( operatorIndex + lengthOfOperator ).trim() ); - String[] attributeName = new String[1]; - String attributeValue; - if ( predicatesArray.length == 1 ) { - attributeName[0] = this.mqttMessage.getTopic(); - attributeValue = this.mqttMessage.getMessage(); - } else { - attributeName[0] = predicatesArray[i].substring( 0, operatorIndex ).trim(); - attributeValue = extractValue( attributeName[0] ); - if ( !validateContent( attributeValue ) ) { - throw new RuntimeException( "Content value was not valid." ); - } - } - algBuilder.values( attributeName, attributeValue ); - RexNode predicateNode = algBuilder.call(predicateOperator, algBuilder.field( attributeName[0] ), comparisionLiteral ); - predicateList.add( predicateNode ); - } - - */ - return predicateList; - } - - List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { try { - // Prepare PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); log.debug( "AlgRoot was prepared." ); - List> rows = result.getRows( statement, -1 ); statement.getTransaction().commit(); return rows; @@ -497,22 +146,4 @@ List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statem } } - - /** - * saves predicatesString in array seperated by commas - * @param predicatesString - * @return - */ - private String[] saveInArray( String predicatesString ) { - if ( predicatesString != null ) { - String[] queryArray = predicatesString.split( "," ); - for ( int i = 0; i < queryArray.length; i++ ) { - queryArray[i] = queryArray[i].trim(); - } - return queryArray; - } else { - return new String[0]; - } - } - -} +} \ No newline at end of file From 202d39a89f88ac267a8d6282162043ccaae63614 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 8 Sep 2023 18:36:52 +0200 Subject: [PATCH 080/114] finished implementing and testing --- .../polypheny/db/stream/StreamProcessor.java | 7 +- .../db/stream/StreamProcessorImpl.java | 39 +++- .../db/mqtt/MqttStreamProcessor.java | 96 ++------- .../db/mqtt/MqttStreamProcessorTest.java | 184 ++++++++++++++++++ 4 files changed, 246 insertions(+), 80 deletions(-) create mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java diff --git a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java index 2eb3681235..5a3f3511e2 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamProcessor.java @@ -21,9 +21,12 @@ */ public interface StreamProcessor { - //TODO: maybe change type to MqttStream? String getStream( ); - boolean validateContent( String stream ); + boolean isNumber ( String value ); + + boolean isBoolean( String value); + + } diff --git a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java index 56167eee15..d109172b7e 100644 --- a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java @@ -18,6 +18,13 @@ import static org.reflections.Reflections.log; +import java.util.List; +import org.polypheny.db.PolyImplementation; +import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.prepare.Context; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.TransactionException; + public class StreamProcessorImpl implements StreamProcessor { String stream; @@ -30,10 +37,36 @@ public String getStream() { return stream; } + public boolean isNumber( String value ) { + try{ + Double.parseDouble( value ); + } catch ( NumberFormatException e ) { + return false; + } + return true; + } - @Override - public boolean validateContent( String stream ) { - return false; + public boolean isBoolean( String value) { + return value.equals( "true" ) || value.equals( "false" ); } + + protected List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement ) { + + try { + PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); + log.debug( "AlgRoot was prepared." ); + List> rows = result.getRows( statement, -1 ); + statement.getTransaction().commit(); + return rows; + } catch ( Throwable e ) { + log.error( "Error during execution of stream processor query", e ); + try { + statement.getTransaction().rollback(); + } catch ( TransactionException transactionException ) { + log.error( "Could not rollback", e ); + } + return null; + } + } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index dfa76e21cf..6c3cf84daa 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -17,14 +17,12 @@ package org.polypheny.db.mqtt; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.bson.BsonArray; +import org.bson.BsonBoolean; import org.bson.BsonDocument; -import org.bson.BsonInt32; +import org.bson.BsonDouble; import org.bson.BsonString; -import org.bson.BsonValue; import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; @@ -36,31 +34,31 @@ import org.polypheny.db.prepare.Context; import org.polypheny.db.prepare.PolyphenyDbCatalogReader; import org.polypheny.db.processing.Processor; -import org.polypheny.db.stream.StreamProcessor; +import org.polypheny.db.stream.StreamProcessorImpl; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.TransactionException; @Slf4j -public class MqttStreamProcessor implements StreamProcessor { +public class MqttStreamProcessor extends StreamProcessorImpl { + private final String filterQuery; - private final StreamProcessor streamProcessor; private final Statement statement; public MqttStreamProcessor( MqttMessage mqttMessage, String filterQuery, Statement statement ) { + super( mqttMessage.getMessage() ); this.filterQuery = filterQuery; - this.streamProcessor = statement.getStreamProcessor( mqttMessage.getMessage() ); this.statement = statement; } + //TODO: in Tutorial schreiben, was allgemein für Strings gilt public boolean applyFilter() { AlgRoot root = processMqlQuery(); - List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + List> res = executeAndTransformPolyAlg( root, statement); log.info( res.toString() ); - return !res.isEmpty(); - + return res.size() != 0; } @@ -73,77 +71,25 @@ private AlgRoot processMqlQuery() { MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "null", this.filterQuery ) ).get( 0 ); String msg = getStream(); + BsonDocument msgDoc; AlgNode input; if ( msg.contains( "{" ) && msg.contains( "}" ) ) { // msg is in JSON format - BsonDocument msgDoc = BsonDocument.parse(msg); - input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); + msgDoc = BsonDocument.parse( msg ); } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { // msg is an array - List values = arrayToBsonList(msg); - BsonDocument msgDoc = new BsonDocument( "$$ROOT", new BsonArray( values ) ); - input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); + msgDoc = BsonDocument.parse( "{\"$$ROOT\":" + msg + "}" ); + } else if ( isNumber( msg ) ) { + double value = Double.parseDouble( msg ); + msgDoc = new BsonDocument( "$$ROOT", new BsonDouble( value ) ); + } else if ( isBoolean( msg ) ) { + boolean value = Boolean.parseBoolean( msg ); + msgDoc = new BsonDocument( "$$ROOT", new BsonBoolean( value ) ); } else { - // msg is single value - BsonDocument msgDoc = new BsonDocument( "$$ROOT", new BsonInt32( Integer.parseInt( msg ) ) ); - input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); + // msg is String + msgDoc = new BsonDocument( "$$ROOT", new BsonString( msg ) ); } - - + input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); return mqlConverter.convert( find, input ); } - - - /** - * converts the array, currently a String into a list of BSON values - * @param msg - */ - private List arrayToBsonList( String msg ) { - msg = msg.replace( "[", "" ).replace( "]", "" ); - String[] array = msg.split( "," ); - List list = new ArrayList<>(array.length); - for ( String stringValue : array ) { - //TODO: if BsonString dont work -> try with BsonInt - BsonString bsonValue = new BsonString( stringValue.trim() ); - list.add( bsonValue ); - } - return list; - } - - - @Override - public String getStream() { - return streamProcessor.getStream(); - } - - - /** - * {@inheritDoc} - */ - @Override - public boolean validateContent( String msg ) { - //TODO: Implement - return true; - } - - - List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { - - try { - PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); - log.debug( "AlgRoot was prepared." ); - List> rows = result.getRows( statement, -1 ); - statement.getTransaction().commit(); - return rows; - } catch ( Throwable e ) { - log.error( "Error during execution of stream processor query", e ); - try { - statement.getTransaction().rollback(); - } catch ( TransactionException transactionException ) { - log.error( "Could not rollback", e ); - } - return null; - } - } - } \ No newline at end of file diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java new file mode 100644 index 0000000000..ae9609b4ed --- /dev/null +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java @@ -0,0 +1,184 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; +import org.bson.BsonDouble; +import org.bson.BsonValue; +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.transaction.Statement; +import org.polypheny.db.transaction.Transaction; + +public class MqttStreamProcessorTest { + + @BeforeClass + public static void init() { + TestHelper testHelper = TestHelper.getInstance(); + + } + + @Test + public void filterTestForSingleNumberMessage() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":10}"; + + MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue(streamProcessor.applyFilter()); + + } + + @Test + public void filterTestForSingleNumberMessage2() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":10}"; + + MqttMessage mqttMessage1 = new MqttMessage( "15", "button/battery" ); + MqttStreamProcessor streamProcessor1 = new MqttStreamProcessor( mqttMessage1, filterQuery, st ); + assertFalse(streamProcessor1.applyFilter()); + + } + + + @Test + public void filterTestForSingleStringMessage1() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":\"shouldMatch\"}"; + + MqttMessage mqttMessage2 = new MqttMessage( "shouldMatch", "button/battery" ); + MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( mqttMessage2, filterQuery, st ); + assertTrue(streamProcessor2.applyFilter()); + + } + + + @Test + public void filterTestForSingleStringMessage2() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":\"shouldNot\"}"; + MqttMessage mqttMessage2 = new MqttMessage( "shouldNotMatch", "button/battery" ); + MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( mqttMessage2, filterQuery, st ); + assertFalse(streamProcessor2.applyFilter()); + } + + + @Test + public void filterTestForArrayMessage() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":10}"; + MqttMessage mqttMessage = new MqttMessage( "[10]", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter() ); + } + + + @Test + public void filterTestForArrayMessage2() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":[10]}"; + MqttMessage mqttMessage = new MqttMessage( "[10]", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter() ); + } + + + @Test + public void filterTestForArrayMessageFalse() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":10}"; + MqttMessage mqttMessage = new MqttMessage( "[15, 14]", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertFalse( streamProcessor.applyFilter()); + } + + @Test + public void filterTestForBooleanMessageTrue() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":true}"; + MqttMessage mqttMessage = new MqttMessage( "true", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertFalse( streamProcessor.applyFilter()); + } + @Test + public void filterTestForBooleanMessageFalse() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"$$ROOT\":true}"; + MqttMessage mqttMessage = new MqttMessage( "false", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertFalse( streamProcessor.applyFilter()); + } + + + @Test + public void filterTestForJsonNumberMessage() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"count\":10}"; + MqttMessage mqttMessage = new MqttMessage( "{\"count\":10}", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter()); + } + + + @Test + public void filterTestForJsonArrayMessage() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"array\":10}"; + MqttMessage mqttMessage = new MqttMessage( "{\"array\":[10]}", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter()); + } + + + @Test + public void filterTestForJsonStringMessage() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"content\":\"online\"}"; + MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter()); + } + + @Test + public void filterTestForJsonStringMessage2() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"content\":\"online\"}"; + MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + assertTrue( streamProcessor.applyFilter()); + } + + +} From 2d5628f55875c32f8cdf971acfb22e861b3a1580 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 8 Sep 2023 18:41:35 +0200 Subject: [PATCH 081/114] Deleted PolyStream class --- .../org/polypheny/db/mqtt/PolyStream.java | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java deleted file mode 100644 index 4fed4544ab..0000000000 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PolyStream.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.mqtt; - -import lombok.Getter; -import lombok.Setter; -import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.NamespaceType; - -public class PolyStream { - - // Representation of 1 content in a namespace - @Getter - final String topic; - @Getter - private final String uniqueNameOfInterface; - @Getter - private final String content; - @Getter - private final String namespace; - @Getter - private final NamespaceType namespaceType; - @Getter - @Setter - private long namespaceID; - @Setter - @Getter - private long databaseId; - @Setter - @Getter - private int userId; - @Getter - @Setter - private long storeID; // the ID of collection/graph/table... the place where info is/should be saved - - - public PolyStream( String topic, String uniqueNameInterface, String content, String namespace, NamespaceType namespaceType ) { - this.topic = topic; - this.uniqueNameOfInterface = uniqueNameInterface; - this.content = content; - this.namespace = namespace; - this.namespaceType = namespaceType; - this.databaseId = Catalog.defaultDatabaseId; - this.userId = Catalog.defaultUserId; - } - - -} From 2380174b35375a0f24af3f340135ace61b66f814 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 8 Sep 2023 18:45:44 +0200 Subject: [PATCH 082/114] reformatted code --- .../polypheny/db/stream/StreamProcessorImpl.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java index d109172b7e..1cfc4fdcda 100644 --- a/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/stream/StreamProcessorImpl.java @@ -21,24 +21,27 @@ import java.util.List; import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgRoot; -import org.polypheny.db.prepare.Context; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.TransactionException; public class StreamProcessorImpl implements StreamProcessor { String stream; - public StreamProcessorImpl(String stream) { + + + public StreamProcessorImpl( String stream ) { this.stream = stream; } + @Override public String getStream() { return stream; } + public boolean isNumber( String value ) { - try{ + try { Double.parseDouble( value ); } catch ( NumberFormatException e ) { return false; @@ -46,7 +49,8 @@ public boolean isNumber( String value ) { return true; } - public boolean isBoolean( String value) { + + public boolean isBoolean( String value ) { return value.equals( "true" ) || value.equals( "false" ); } @@ -69,4 +73,5 @@ protected List> executeAndTransformPolyAlg( AlgRoot algRoot, Statem return null; } } + } From 38b1528308e7bd80d268737a82d36f2c6dd5cb77 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 10 Sep 2023 17:48:09 +0200 Subject: [PATCH 083/114] made various changes - implemented new query form - ssl connection ongoing - collection name special characters beachtet - Klassen variablen umbenannt - Methoden umbenannt --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 555 +++++++++--------- 1 file changed, 270 insertions(+), 285 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 68000001d7..55b9befd78 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -23,21 +23,27 @@ import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; +import java.security.KeyFactory; import java.security.KeyStore; -import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -96,10 +102,11 @@ public void start() { Map mqttDefaultSettings = new HashMap<>(); mqttDefaultSettings.put( "broker", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); - mqttDefaultSettings.put( "namespace", "wohnzimmer" ); + mqttDefaultSettings.put( "Tsl/SslConnection", "false" ); + mqttDefaultSettings.put( "namespace", "default" ); mqttDefaultSettings.put( "namespaceType", "DOCUMENT" ); - mqttDefaultSettings.put( "collectionPerTopic", "TRUE" ); - mqttDefaultSettings.put( "collectionName", "default" ); + mqttDefaultSettings.put( "commonCollection", "false" ); + mqttDefaultSettings.put( "collectionName", "" ); mqttDefaultSettings.put( "Query Interface Name", "mqtt" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -120,18 +127,9 @@ public static class MqttStreamServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_DESCRIPTION = "Connection establishment to a MQTT broker."; @SuppressWarnings("WeakerAccess") - public static final List AVAILABLE_SETTINGS = ImmutableList.of( - new QueryInterfaceSettingString( "broker", false, true, false, null ), - new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), - new QueryInterfaceSettingString( "namespace", false, true, true, null ), - // "RELATIONAL", "GRAPH" type are not supported yet. - new QueryInterfaceSettingList( "namespaceType", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), - new QueryInterfaceSettingList( "collectionPerTopic", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), - new QueryInterfaceSettingString( "collectionName", true, false, true, null ), - new QueryInterfaceSettingString( "topics", false, true, true, null ), - new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), - new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) - ); + public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, null ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), new QueryInterfaceSettingString( "namespace", false, true, true, null ), + // "RELATIONAL", "GRAPH" types are not supported yet. + new QueryInterfaceSettingList( "namespaceType", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "collectionName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); @Getter private final String broker; @@ -149,11 +147,11 @@ public static class MqttStreamServer extends QueryInterface { private Mqtt3AsyncClient client; private String namespaceName; private NamespaceType namespaceType; - private AtomicBoolean collectionPerTopic; + private AtomicBoolean commonCollection; private String collectionName; private final long databaseId; private final int userId; - final boolean ssl; + final boolean ssl; // todo this. boolean createCommonCollection = false; private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -179,11 +177,11 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.namespaceName = name; this.namespaceType = type; - this.collectionPerTopic = new AtomicBoolean( Boolean.parseBoolean( settings.get( "collectionPerTopic" ) ) ); - this.collectionName = settings.get( "collectionName" ) == null ? settings.get( "collectionName" ) : settings.get( "collectionName" ).trim(); - if ( !this.collectionPerTopic.get() ) { - if ( (this.collectionName.equals( null ) | this.collectionName.equals( "" )) ) { - throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + this.commonCollection = new AtomicBoolean( Boolean.parseBoolean( settings.get( "commonCollection" ) ) ); + this.collectionName = settings.get( "collectionName" ) == null ? settings.get( "collectionName" ) : settings.get( "collectionName" ).trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + if ( this.commonCollection.get() ) { + if ( this.collectionName.equals( null ) || this.collectionName.isEmpty() || this.collectionName.isBlank() ) { + throw new NullPointerException( "commonCollection is set to FALSE but no valid collection name was given! Please enter a collection name." ); } else if ( !collectionExists( this.collectionName ) ) { this.createCommonCollection = true; createStreamCollection( this.collectionName ); @@ -202,79 +200,25 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au @Override public void run() { if ( ssl ) { - //TODO: look at book: essentials - KeyStore keyStore = null; - try { - keyStore = KeyStore.getInstance( "PKCS12" ); - } catch ( KeyStoreException e ) { - throw new RuntimeException( e ); - } - //https://community.hivemq.com/t/sslwithdefaultconfig/946/4 - // load default jvm keystore - //todo: this.getUniqueName() - String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; - try { - keyStore.load( new FileInputStream( PolyphenyHomeDirManager.getInstance().getFileIfExists( path ) ), "".toCharArray() ); - } catch ( IOException e ) { - throw new RuntimeException( e ); - } catch ( NoSuchAlgorithmException e ) { - throw new RuntimeException( e ); - } catch ( CertificateException e ) { - throw new RuntimeException( e ); - } - TrustManagerFactory tmf = null; - try { - tmf = TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm() ); - } catch ( NoSuchAlgorithmException e ) { - throw new RuntimeException( e ); - } - - try { - tmf.init( keyStore ); - } catch ( KeyStoreException e ) { - throw new RuntimeException( e ); - } - - this.client = MqttClient.builder() - .useMqttVersion3() - .identifier( getUniqueName() ) - .serverHost( broker ) - .serverPort( brokerPort ) + this.client = MqttClient.builder().useMqttVersion3().identifier( getUniqueName() ).serverHost( broker ).serverPort( brokerPort ) .sslConfig() - //.keyManagerFactory(kmf) - .trustManagerFactory( tmf ) - .applySslConfig() - .useSslWithDefaultConfig() - - .buildAsync(); + //TODO: delete or enter password from GUI password thinghere and in method + .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ).trustManagerFactory( SslHelper.createTrustManagerFactory( "ca.crt" ) ).applySslConfig().buildAsync(); } else { - this.client = MqttClient.builder() - .useMqttVersion3() - .identifier( getUniqueName() ) - .serverHost( broker ) - .serverPort( brokerPort ) - .buildAsync(); - } - - client.connectWith() - //.simpleAuth() - //.username("") - //.password("my-password".getBytes()) - //.applySimpleAuth() - .send() - .whenComplete( ( connAck, throwable ) -> { - if ( throwable != null ) { - throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); - } else { - log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); - subscribe( toList( this.settings.get( "topics" ) ) ); - } - } - ); + this.client = MqttClient.builder().useMqttVersion3().identifier( getUniqueName() ).serverHost( broker ).serverPort( brokerPort ).buildAsync(); + } + + client.connectWith().send().whenComplete( ( connAck, throwable ) -> { + if ( throwable != null ) { + throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); + } else { + log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); + subscribe( toList( this.settings.get( "topics" ) ) ); + } + } ); } @@ -289,14 +233,13 @@ public List getAvailableSettings() { public void shutdown() { client.disconnect().whenComplete( ( disconn, throwable ) -> { - if ( throwable != null ) { - throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ". Please try again.", throwable ); - } else { - log.info( "{} stopped.", INTERFACE_NAME ); - monitoringPage.remove(); - } - } - ); + if ( throwable != null ) { + throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ". Please try again.", throwable ); + } else { + log.info( "{} stopped.", INTERFACE_NAME ); + monitoringPage.remove(); + } + } ); } @@ -380,61 +323,38 @@ protected void reloadSettings( List updatedSettings ) { break; case "namespace": - /* - try { - this.wait(2000); - } catch ( InterruptedException e ) { - throw new RuntimeException( e ); - } - this.notify(); - - */ - //TODO current thread is not owner Exception! -> dann wird nichts mehr aus dem Block ausgeführt! - /* - try { - client.wait(); - } catch ( InterruptedException e ) { - throw new RuntimeException( e ); - } - - - */ - synchronized ( this.namespaceName ) { - String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); - if ( updatedSettings.contains( "namespaceType" ) ) { - if ( updatedSettings.indexOf( "namespaceType" ) < updatedSettings.indexOf( "namespace" ) ) { - NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespaceType" ) ); - try { - if ( !namespaceExists( newNamespaceName, type ) ) { - createNamespace( newNamespaceName, type ); - } - this.namespaceName = newNamespaceName; - this.namespaceType = type; - createAllCollections(); - } catch ( RuntimeException e ) { - this.settings.put( "namespace", this.namespaceName ); - this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); - throw new RuntimeException( e ); - } - } // else checking for namespace happens in case "namespaceType" - } else { + String newNamespaceName = this.getCurrentSettings().get( "namespace" ).trim(); + if ( updatedSettings.contains( "namespaceType" ) ) { + if ( updatedSettings.indexOf( "namespaceType" ) < updatedSettings.indexOf( "namespace" ) ) { + NamespaceType type = NamespaceType.valueOf( this.getCurrentSettings().get( "namespaceType" ) ); try { - if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { - createNamespace( newNamespaceName, this.namespaceType ); + if ( !namespaceExists( newNamespaceName, type ) ) { + createNamespace( newNamespaceName, type ); } this.namespaceName = newNamespaceName; + this.namespaceType = type; createAllCollections(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); + this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); } + } // else checking for namespace happens in case "namespaceType" + } else { + try { + if ( !namespaceExists( newNamespaceName, this.namespaceType ) ) { + createNamespace( newNamespaceName, this.namespaceType ); + } + this.namespaceName = newNamespaceName; + createAllCollections(); + } catch ( RuntimeException e ) { + this.settings.put( "namespace", this.namespaceName ); + throw new RuntimeException( e ); } } break; - case "namespaceType": NamespaceType newNamespaceType = NamespaceType.valueOf( this.getCurrentSettings().get( "namespaceType" ) ); - //synchronized ( this.namespaceType ) { if ( updatedSettings.contains( "namespace" ) ) { if ( updatedSettings.indexOf( "namespace" ) < updatedSettings.indexOf( "namespaceType" ) ) { String newName = this.getCurrentSettings().get( "namespace" ); @@ -463,24 +383,22 @@ protected void reloadSettings( List updatedSettings ) { throw new RuntimeException( e ); } } - - // } - break; - case "collectionPerTopic": - this.collectionPerTopic.set( Boolean.parseBoolean( this.getCurrentSettings().get( "collectionPerTopic" ) ) ); + case "commonCollection": + this.commonCollection.set( Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ) ); createAllCollections(); break; case "collectionName": String newCollectionName = this.getCurrentSettings().get( "collectionName" ).trim(); + newCollectionName = newCollectionName == null ? null : newCollectionName.trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); boolean mode; - if ( updatedSettings.contains( "collectionPerTopic" ) ) { - mode = Boolean.parseBoolean( this.getCurrentSettings().get( "collectionPerTopic" ) ); + if ( updatedSettings.contains( "commonCollection" ) ) { + mode = Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ); } else { - mode = this.collectionPerTopic.get(); + mode = this.commonCollection.get(); } - if ( !mode ) { - if ( !(newCollectionName.equals( "null" ) | newCollectionName.equals( "" )) ) { + if ( mode ) { + if ( !(newCollectionName.equals( "null" ) || newCollectionName.isEmpty() || newCollectionName.isBlank()) ) { if ( !collectionExists( newCollectionName ) ) { createStreamCollection( this.collectionName ); } @@ -488,7 +406,7 @@ protected void reloadSettings( List updatedSettings ) { createAllCollections(); } else { this.settings.put( "collectionName", this.collectionName ); - throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + throw new NullPointerException( "commonCollection is set to FALSE but no valid collection name was given! Please enter a collection name." ); } } else { @@ -496,36 +414,9 @@ protected void reloadSettings( List updatedSettings ) { } break; case "filterQuery": - //TODO: Problem mit Komma in Query und trennen der Queries nach komma String queryString = this.getCurrentSettings().get( "filterQuery" ); - if ( queryString == null || queryString.isBlank() ) { - filterMap.clear(); - } else { - String[] queryArray = queryString.split( "," ); - for ( String query : queryArray ) { - int separatorIndex = query.indexOf( ":" ); - String topic = query.substring( 0, separatorIndex ).trim(); - String mqlQuery = query.substring( separatorIndex + 1 ).trim(); - if ( filterMap.containsKey( topic ) && !filterMap.get( topic ).equals( mqlQuery ) ) { - filterMap.replace( topic, mqlQuery ); - } else { - filterMap.put( topic, mqlQuery ); - } - } - - for ( Entry entry : filterMap.entrySet() ) { - boolean remove = true; - for ( String query : queryArray ) { - if ( query.startsWith( entry.getKey() ) ) { - remove = false; - break; - } - } - if ( remove ) { - filterMap.remove( entry.getKey(), entry.getValue() ); - } - } - } + filterMap.clear(); + saveQueriesInMap( queryString ); break; } } @@ -543,7 +434,7 @@ void subscribe( List newTopics ) { /** * subscribes to one given topic and adds it to the List topics. * - * @param topic: the topic the client should subscribe to. + * @param topic the topic the client should subscribe to. */ public void subscribe( String topic ) { client.subscribeWith().topicFilter( topic ).callback( subMsg -> { @@ -569,7 +460,7 @@ public void subscribe( String topic ) { throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topicsMap.put( topic, new AtomicLong( 0 ) ); - if ( this.collectionPerTopic.get() && !collectionExists( topic ) ) { + if ( !this.commonCollection.get() && !collectionExists( topic ) ) { createStreamCollection( topic ); } } @@ -608,7 +499,7 @@ void processMsg( Mqtt3Publish subMsg ) { String message = extractPayload( subMsg ); String wildcardTopic = ""; if ( !topicsMap.containsKey( topic ) ) { - wildcardTopic = compareWithWildcardsTopic( topic ); + wildcardTopic = getWildcardTopic( topic ); topicsMap.get( wildcardTopic ).incrementAndGet(); } else { topicsMap.get( topic ).incrementAndGet(); @@ -618,36 +509,33 @@ void processMsg( Mqtt3Publish subMsg ) { if ( this.filterMap.containsKey( topic ) ) { String filterQuery = this.filterMap.get( topic ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); - // false is returned when message should not be stored in DB - if ( streamProcessor.processStream() ) { - insert(mqttMessage, transaction); + // false is returned when a message should not be stored in DB + if ( streamProcessor.applyFilter() ) { + insert( mqttMessage, transaction ); + } + } else if ( !wildcardTopic.isEmpty() && this.filterMap.containsKey( wildcardTopic ) ) { + String filterQuery = this.filterMap.get( wildcardTopic ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); + if ( streamProcessor.applyFilter() ) { + insert( mqttMessage, transaction ); } } else { - insert(mqttMessage, transaction); + insert( mqttMessage, transaction ); } } - private void insert ( MqttMessage mqttMessage, Transaction transaction) { - //TODO: besserer Name für message objekt: was mit insert + private void insert( MqttMessage mqttMessage, Transaction transaction ) { ReceivedMqttMessage receivedMqttMessage; synchronized ( settingsLock ) { - if ( this.collectionPerTopic.get() ) { + if ( !this.commonCollection.get() ) { String collectionToBeSaved; - /* - if ( wildcardTopic != null || wildcardTopic.isBlank() ) { - - } else { - collectionToBeSaved = wildcardTopic; - } - */ - collectionToBeSaved = mqttMessage.getTopic(); + collectionToBeSaved = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); } else { receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); } } - StreamCapture streamCapture = new StreamCapture( transaction ); streamCapture.insert( receivedMqttMessage ); } @@ -665,23 +553,42 @@ private static String extractPayload( Mqtt3Publish subMsg ) { * @param queries */ private void saveQueriesInMap( String queries ) { - List queriesList = toList( queries ); - for ( String topicQueryString : queriesList ) { - int separatorIndex = topicQueryString.indexOf( ":" ); - String topic = topicQueryString.substring( 0, separatorIndex ); - String query = topicQueryString.substring( separatorIndex + 1 ); - if ( this.filterMap.containsKey( topic ) ) { - String val = this.filterMap.get( topic ); - // TODO: check: now also or can be done in query so no several queries for one topic: replace old query - this.filterMap.replace( topic, query ); - } else { - this.filterMap.put( topic, query ); + Stack brackets = new Stack<>(); + String query; + while ( !queries.isBlank() ) { + int index = 0; + String topic = queries.substring( 0, queries.indexOf( ":" ) ); + queries = queries.replace( topic + ":", "" ); + while ( !queries.isBlank() ) { + char c = queries.charAt( index ); + if ( c == '{' ) { + brackets.push( c ); + index++; + } else if ( c == '}' ) { + if ( brackets.pop().equals( '{' ) ) { + if ( brackets.isEmpty() ) { + query = queries.substring( 0, index + 1 ); + if ( this.filterMap.containsKey( topic ) && !this.filterMap.get( topic ).equals( queries ) ) { + this.filterMap.replace( topic, query ); + } else { + this.filterMap.put( topic, query ); + } + queries = queries.substring( index + 1 ); + break; + } + } else { + throw new RuntimeException( String.format( "The brackets in the query to the topic %s are not set correctly!", topic ) ); + } + } + if ( index < queries.toCharArray().length ) { + index++; + } } } } - private String compareWithWildcardsTopic( String topic ) { + private String getWildcardTopic( String topic ) { for ( String t : topicsMap.keySet() ) { //multilevel wildcard if ( t.contains( "#" ) && topic.startsWith( t.substring( 0, t.indexOf( "#" ) ) ) ) { @@ -720,6 +627,7 @@ public List toList( String string ) { * @return true: collection already exists, false: collection does not exist. */ private boolean collectionExists( String collectionName ) { + collectionName = collectionName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); Catalog catalog = Catalog.getInstance(); Pattern pattern = new Pattern( collectionName ); List collectionList = null; @@ -731,6 +639,7 @@ private boolean collectionExists( String collectionName ) { private void createStreamCollection( String collectionName ) { + collectionName = collectionName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); long namespaceID; @@ -740,13 +649,8 @@ private void createStreamCollection( String collectionName ) { try { List dataStores = new ArrayList<>(); //TODO: StreamCollection einbinden - DdlManager.getInstance().createCollection( - namespaceID, - collectionName, - true, //only creates collection if it does not already exist. - dataStores.size() == 0 ? null : dataStores, - PlacementType.MANUAL, - statement ); + DdlManager.getInstance().createCollection( namespaceID, collectionName, true, //only creates collection if it does not already exist. + dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); transaction.commit(); } catch ( EntityAlreadyExistsException | TransactionException e ) { throw new RuntimeException( "Error while creating a new collection:", e ); @@ -758,19 +662,19 @@ private void createStreamCollection( String collectionName ) { private void createAllCollections() { synchronized ( settingsLock ) { - if ( this.collectionPerTopic.get() ) { + if ( !this.commonCollection.get() ) { for ( String t : this.topicsMap.keySet() ) { if ( !collectionExists( t ) ) { createStreamCollection( t ); } } } else { - if ( !(this.collectionName.equals( "null" ) | this.collectionName.equals( "" )) ) { + if ( !(this.collectionName.equals( null ) || this.collectionName.equals( "" ) || this.collectionName.isBlank()) ) { if ( !collectionExists( this.collectionName ) ) { createStreamCollection( this.collectionName ); } } else { - throw new NullPointerException( "collectionPerTopic is set to FALSE but no valid collection name was given! Please enter a collection name." ); + throw new NullPointerException( "commonCollection is set to 'true' but no valid collection name was given! Please enter a collection name." ); } } } @@ -818,15 +722,137 @@ public String getInterfaceType() { } + private static class SslHelper { + + private KeyFactory getKeyFactoryInstance() { + try { + return KeyFactory.getInstance( "RSA" ); + } catch ( NoSuchAlgorithmException e ) { + throw new RuntimeException( e ); + } + } + + + private X509Certificate createX509CertificateFromFile( final String certificateFileName ) { + String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + certificateFileName; + final File file = PolyphenyHomeDirManager.getInstance().getFileIfExists( path ); + if ( !file.isFile() ) { + throw new RuntimeException( String.format( "The certificate file %s doesn't exist.", certificateFileName ) ); + } + final X509Certificate certificate; + try { + final CertificateFactory certificateFactoryX509 = CertificateFactory.getInstance( "X.509" ); + final InputStream inputStream = new FileInputStream( file ); + certificate = (X509Certificate) certificateFactoryX509.generateCertificate( inputStream ); + inputStream.close(); + } catch ( Exception e ) { + throw new RuntimeException( e ); + } + + return certificate; + } + + + /* + private PrivateKey createPrivateKeyFromPemFile(final String keyFileName) { + final PrivateKey privateKey; + try { + final PemReader pemReader = new PemReader( new FileReader( keyFileName ) ); + final PemObject pemObject = pemReader.readPemObject(); + final byte[] pemContent = pemObject.getContent(); + pemReader.close(); + final PKCS8EncodedKeySpec encodedKeySpec = new PKCS8EncodedKeySpec( pemContent ); + final KeyFactory keyFactory = getKeyFactoryInstance(); + privateKey = keyFactory.generatePrivate( encodedKeySpec ); + } catch ( Exception e ) { + throw new RuntimeException(e); + } + return privateKey; + } + */ + private static KeyManagerFactory createKeyManagerFactory( String clientCertificateFileName, String clientKeyFileName, final String clientKeyPassword ) { + final KeyManagerFactory keyManagerFactory; + try { + //todo: this.getUniqueName() in path + String clientCrtPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; + + // load client certificate: + File crtFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( clientCrtPath ); + InputStream crtStream = new FileInputStream( crtFile ); + //CertificateFactory cf = CertificateFactory.getInstance("X.509"); + //Certificate clientCertificate = cf.generateCertificate( crtStream ); + + final KeyStore ks = KeyStore.getInstance( KeyStore.getDefaultType() ); + ks.load( crtStream, null ); + Certificate clientCertificate = ks.getCertificate( "alias" ); + crtStream.close(); + + //load client key: + String keyPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.key"; + File keyFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( keyPath ); + InputStream keyStream = new FileInputStream( keyFile ); + String key = keyStream.toString(); + key = key.replace( "-----BEGIN PRIVATE KEY-----", "" ).replace( "-----END PRIVATE KEY-----", "" ); + final PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec( key.getBytes() ); + KeyFactory kf = KeyFactory.getInstance( "RSA" ); + PrivateKey privateKey = kf.generatePrivate( spec ); + + // create keystore + final KeyStore keyStore = KeyStore.getInstance( KeyStore.getDefaultType() ); + keyStore.load( null, null ); + + keyStore.setCertificateEntry( "certificate", clientCertificate ); + keyStore.setKeyEntry( "private-key", privateKey, "".toCharArray(), new Certificate[]{ clientCertificate } ); + keyManagerFactory = KeyManagerFactory.getInstance( KeyManagerFactory.getDefaultAlgorithm() ); + keyManagerFactory.init( keyStore, "".toCharArray() ); + + } catch ( Exception e ) { + throw new RuntimeException( e ); + } + return keyManagerFactory; + } + + + private static TrustManagerFactory createTrustManagerFactory( final String caCertificateFileName ) { + final TrustManagerFactory trustManagerFactory; + try { + // load ca certificate: + //todo: this.getUniqueName() in path + String caPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "ca.crt"; + File caFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( caPath ); + InputStream ca = new FileInputStream( caFile ); + CertificateFactory cf = CertificateFactory.getInstance( "X.509" ); + Certificate caCertificate = cf.generateCertificate( ca ); + ca.close(); + + final KeyStore trustStore = KeyStore.getInstance( KeyStore.getDefaultType() ); + trustStore.load( null, null ); + trustStore.setCertificateEntry( "ca-certificate", caCertificate ); + trustManagerFactory = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); + trustManagerFactory.init( trustStore ); + } catch ( Exception e ) { + throw new RuntimeException( e ); + } + return trustManagerFactory; + } + + } + + private class MonitoringPage { private final InformationPage informationPage; private final InformationGroup informationGroupTopics; + private final InformationGroup informationGroupInfo; + private final InformationGroup informationGroupReceivedMessages; + private final InformationGroup informationGroupPub; + private final InformationGroup informationGroupReconn; private final InformationTable topicsTable; private final InformationTable messageTable; private final InformationKeyValue brokerKv; private final InformationAction reconnButton; + private final InformationAction msgButton; public MonitoringPage() { @@ -836,30 +862,24 @@ public MonitoringPage() { informationPage.setRefreshFunction( this::update ); im.addPage( informationPage ); - InformationGroup informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); + informationGroupInfo = new InformationGroup( informationPage, "Information" ).setOrder( 1 ); im.addGroup( informationGroupInfo ); brokerKv = new InformationKeyValue( informationGroupInfo ); im.registerInformation( brokerKv ); informationGroupTopics = new InformationGroup( informationPage, "Subscribed Topics" ).setOrder( 2 ); im.addGroup( informationGroupTopics ); - topicsTable = new InformationTable( - informationGroupTopics, - List.of( "Topic", "Number of received messages" ) - ); + topicsTable = new InformationTable( informationGroupTopics, List.of( "Topic", "Number of received messages" ) ); im.registerInformation( topicsTable ); - InformationGroup informationGroupMessage = new InformationGroup( informationPage, "Recently received messages" ).setOrder( 2 ); - im.addGroup( informationGroupMessage ); - messageTable = new InformationTable( - informationGroupMessage, - List.of( "Topic", "Message" ) - ); + informationGroupReceivedMessages = new InformationGroup( informationPage, "Recently received messages" ).setOrder( 2 ); + im.addGroup( informationGroupReceivedMessages ); + messageTable = new InformationTable( informationGroupReceivedMessages, List.of( "Topic", "Message" ) ); im.registerInformation( messageTable ); - InformationGroup informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); + informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupPub ); - InformationAction msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { + msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; try { client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); @@ -871,7 +891,7 @@ public MonitoringPage() { im.registerInformation( msgButton ); // Reconnection button - InformationGroup informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); + informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); im.addGroup( informationGroupReconn ); reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { String end = "Reconnecting to broker"; @@ -882,67 +902,23 @@ public MonitoringPage() { client.toBlocking().disconnect(); run(); update(); - /* - client.disconnect().whenComplete( ( disconn, throwable ) -> { - if ( throwable != null ) { - throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ".", throwable ); - } else { - run(); - update(); - } - } - ); - */ } return end; } ); im.registerInformation( reconnButton ); - - // Test button - InformationGroup informationGroupTest = new InformationGroup( informationPage, "Test implementation with message: '10', topic:'button' and filterQuery:'value:10'." ).setOrder( 6 ); - im.addGroup( informationGroupTest ); - InformationAction testButton = new InformationAction( informationGroupTest, "Call MqttProcessor.processStream", ( parameters ) -> { - Transaction transaction = getTransaction(); - MqttMessage mqttMessage = new MqttMessage( "10", "button" ); - addMessageToQueue( "button", "10" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, "value:10", transaction.createStatement() ); - if ( streamProcessor.processStream() ) { - log.info( "Test returned true!" ); - } else { - log.info( "Test returned false." ); - } - return ""; - } ); - im.registerInformation( testButton ); - - } public void update() { - /* TODO : rmv concurency test - String s = namespace; - String c; - for ( ;; ) { - synchronized ( namespace ) { - if ( s.equals( namespace ) ) { - c = "!"; - } else { - c = "."; - } - } - - System.out.print(c); - } -*/ topicsTable.reset(); if ( topicsMap.isEmpty() ) { topicsTable.addRow( "No topic subscriptions" ); } else { for ( Entry t : topicsMap.entrySet() ) { - topicsTable.addRow( t.getKey(), t.getValue() ); + String filterQuery = filterMap.get( t.getKey() ); + topicsTable.addRow( t.getKey(), t.getValue(), filterQuery == null ? "" : filterQuery ); } } @@ -961,14 +937,23 @@ public void update() { brokerKv.putPair( "Client state", client.getState() + "" ); brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get() + "" ); //TODO: check this after having SSL Configuration. - brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); + brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig().get() + "" ); } public void remove() { InformationManager im = InformationManager.getInstance(); im.removeInformation( topicsTable ); + im.removeInformation( brokerKv ); + im.removeInformation( messageTable ); + im.removeInformation( msgButton ); + im.removeInformation( reconnButton ); + im.removeGroup( informationGroupTopics ); + im.removeGroup( informationGroupInfo ); + im.removeGroup( informationGroupReconn ); + im.removeGroup( informationGroupPub ); + im.removeGroup( informationGroupReceivedMessages ); im.removePage( informationPage ); } From baf6eb3bc25707cd163b720b6e7a6f4bf10e4dee Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 10 Sep 2023 17:48:45 +0200 Subject: [PATCH 084/114] cleaned up code --- .../org/polypheny/db/mqtt/StreamCapture.java | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 45e0c2f5e2..ece69b616c 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -16,7 +16,6 @@ package org.polypheny.db.mqtt; -import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.bson.BsonDocument; @@ -47,13 +46,12 @@ public class StreamCapture { public void insert( ReceivedMqttMessage receivedMqttMessage ) { this.receivedMqttMessage = receivedMqttMessage; - insertDocument( this.receivedMqttMessage.getCollectionName() ); + insertMessage(); } - // added by Datomo - public void insertDocument( String collectionName) { - String sqlCollectionName = this.receivedMqttMessage.getNamespaceName() + "." + collectionName; + private void insertMessage() { + String sqlCollectionName = this.receivedMqttMessage.getNamespaceName() + "." + this.receivedMqttMessage.getCollectionName(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed @@ -77,27 +75,6 @@ public void insertDocument( String collectionName) { } - // added by Datomo - public List scanCollection( String namespaceName, String collectionName ) { - String sqlCollectionName = namespaceName + "." + collectionName; - Statement statement = transaction.createStatement(); - - // Builder which allows to construct the algebra tree which is equivalent to query and is executed - AlgBuilder builder = AlgBuilder.create( statement ); - - AlgNode algNode = builder.docScan( statement, sqlCollectionName ).build(); - - // we can then wrap the tree in an AlgRoot and execute it - AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); - List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); - List result = new ArrayList<>(); - for ( List objectsList : res ) { - result.add( objectsList.get( 0 ).toString() ); - } - return result; - } - - List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { try { From da2f9c8599f1ed1e514580365c574a08874d5e8b Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 10 Sep 2023 17:49:03 +0200 Subject: [PATCH 085/114] added mockito --- plugins/mqtt-stream/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index b02282606a..902b5295cf 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -25,6 +25,7 @@ dependencies { testImplementation project(":plugins:mql-language") testCompileOnly group: 'org.pf4j', name: 'pf4j', version: pf4jVersion testImplementation group: "junit", name: "junit", version: junit_version + testImplementation group: "org.mockito", name: "mockito-core", version: mockito_core_version // --- Test Compile --- From 6075479cf372963ad64b5403a75cf03413507793 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 12 Sep 2023 21:24:51 +0200 Subject: [PATCH 086/114] various changes, see description ssl with bouncy castle from mqtt 3 zu 5 testing von MqttStreamServer angefangen --- plugins/mqtt-stream/build.gradle | 2 + .../polypheny/db/mqtt/MqttStreamPlugin.java | 174 ++++++++++++------ .../db/mqtt/MqttStreamClientTest.java | 139 ++++++++++++++ .../java/org/polypheny/db/mqtt/MqttTest.java | 40 ---- 4 files changed, 260 insertions(+), 95 deletions(-) create mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java delete mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 902b5295cf..46e5db7d92 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -14,6 +14,8 @@ dependencies { // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: hivemq_mqttclient_version + // https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on + implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.70' implementation group: "org.mongodb", name: "mongodb-driver-sync", version: mongodb_driver_sync_version // Apache 2.0 diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 55b9befd78..e4dc25640a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -21,8 +21,11 @@ import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; +import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient; +import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.io.File; import java.io.FileInputStream; +import java.io.FileReader; import java.io.InputStream; import java.nio.charset.Charset; import java.security.KeyFactory; @@ -47,6 +50,9 @@ import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.util.io.pem.PemObject; +import org.bouncycastle.util.io.pem.PemReader; import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; @@ -100,13 +106,13 @@ public MqttStreamPlugin( PluginWrapper wrapper ) { public void start() { // Add MQTT stream Map mqttDefaultSettings = new HashMap<>(); - mqttDefaultSettings.put( "broker", "localhost" ); + mqttDefaultSettings.put( "brokerAddress", "localhost" ); mqttDefaultSettings.put( "brokerPort", "1883" ); mqttDefaultSettings.put( "Tsl/SslConnection", "false" ); mqttDefaultSettings.put( "namespace", "default" ); mqttDefaultSettings.put( "namespaceType", "DOCUMENT" ); mqttDefaultSettings.put( "commonCollection", "false" ); - mqttDefaultSettings.put( "collectionName", "" ); + mqttDefaultSettings.put( "commonCollectionName", "" ); mqttDefaultSettings.put( "Query Interface Name", "mqtt" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); } @@ -127,12 +133,21 @@ public static class MqttStreamServer extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_DESCRIPTION = "Connection establishment to a MQTT broker."; @SuppressWarnings("WeakerAccess") - public static final List AVAILABLE_SETTINGS = ImmutableList.of( new QueryInterfaceSettingString( "broker", false, true, false, null ), new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), new QueryInterfaceSettingString( "namespace", false, true, true, null ), + public static final List AVAILABLE_SETTINGS = ImmutableList.of( + new QueryInterfaceSettingString( "brokerAddress", false, true, false, null ), + new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), + new QueryInterfaceSettingString( "namespace", false, true, true, null ), // "RELATIONAL", "GRAPH" types are not supported yet. - new QueryInterfaceSettingList( "namespaceType", false, true, true, new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "collectionName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); + new QueryInterfaceSettingList( "namespaceType", false, true, true, + new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), + new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), + new QueryInterfaceSettingString( "commonCollectionName", true, false, true, null ), + new QueryInterfaceSettingString( "topics", false, true, true, null ), + new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), + new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); @Getter - private final String broker; + private final String brokerAddress; @Getter private final int brokerPort; private Map topicsMap = new ConcurrentHashMap<>(); @@ -142,13 +157,19 @@ public static class MqttStreamServer extends QueryInterface { * be inserted. If there is no predicate for a topic, then all messages are saved. * If there are several queries for the topic of the message, they are comma seperated. */ + @Getter private Map filterMap = new ConcurrentHashMap<>(); + @Getter private ConcurrentLinkedQueue messageQueue = new ConcurrentLinkedQueue<>(); - private Mqtt3AsyncClient client; + private Mqtt5AsyncClient client; + @Getter private String namespaceName; + @Getter private NamespaceType namespaceType; + @Getter private AtomicBoolean commonCollection; - private String collectionName; + @Getter + private String commonCollectionName; private final long databaseId; private final int userId; final boolean ssl; // todo this. @@ -161,7 +182,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); // Add information page this.monitoringPage = new MonitoringPage(); - this.broker = settings.get( "broker" ).trim(); + this.brokerAddress = settings.get( "brokerAddress" ).trim(); this.brokerPort = Integer.parseInt( settings.get( "brokerPort" ).trim() ); this.databaseId = Catalog.defaultDatabaseId; this.userId = Catalog.defaultUserId; @@ -178,13 +199,18 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.namespaceType = type; this.commonCollection = new AtomicBoolean( Boolean.parseBoolean( settings.get( "commonCollection" ) ) ); - this.collectionName = settings.get( "collectionName" ) == null ? settings.get( "collectionName" ) : settings.get( "collectionName" ).trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + this.commonCollectionName = settings.get( "commonCollectionName" ) == null ? + settings.get( "commonCollectionName" ) : settings.get( "commonCollectionName" ) + .trim() + .replace( '#', '_' ) + .replace( '+', '_' ) + .replace( '/', '_' ); if ( this.commonCollection.get() ) { - if ( this.collectionName.equals( null ) || this.collectionName.isEmpty() || this.collectionName.isBlank() ) { - throw new NullPointerException( "commonCollection is set to FALSE but no valid collection name was given! Please enter a collection name." ); - } else if ( !collectionExists( this.collectionName ) ) { + if ( this.commonCollectionName == null || this.commonCollectionName.isEmpty() || this.commonCollectionName.isBlank() ) { + throw new NullPointerException( "commonCollection is set to true but no valid collection name was given! Please enter a collection name." ); + } else if ( !collectionExists( this.commonCollectionName ) ) { this.createCommonCollection = true; - createStreamCollection( this.collectionName ); + createStreamCollection( this.commonCollectionName ); } } String queryString = settings.get( "filterQuery" ); @@ -201,21 +227,29 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au public void run() { if ( ssl ) { - this.client = MqttClient.builder().useMqttVersion3().identifier( getUniqueName() ).serverHost( broker ).serverPort( brokerPort ) - + this.client = MqttClient.builder().useMqttVersion5() + .identifier( getUniqueName() ) + .serverHost( brokerAddress ) + .serverPort( brokerPort ) .sslConfig() //TODO: delete or enter password from GUI password thinghere and in method - .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ).trustManagerFactory( SslHelper.createTrustManagerFactory( "ca.crt" ) ).applySslConfig().buildAsync(); - + .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ) + .trustManagerFactory( SslHelper.createTrustManagerFactory( "ca.crt" ) ) + .applySslConfig() + .buildAsync(); } else { - this.client = MqttClient.builder().useMqttVersion3().identifier( getUniqueName() ).serverHost( broker ).serverPort( brokerPort ).buildAsync(); + this.client = MqttClient.builder().useMqttVersion5() + .identifier( getUniqueName() ) + .serverHost( brokerAddress ) + .serverPort( brokerPort ) + .buildAsync(); } client.connectWith().send().whenComplete( ( connAck, throwable ) -> { if ( throwable != null ) { throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); } else { - log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, broker, brokerPort ); + log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, brokerAddress, brokerPort ); subscribe( toList( this.settings.get( "topics" ) ) ); } } ); @@ -234,7 +268,7 @@ public void shutdown() { client.disconnect().whenComplete( ( disconn, throwable ) -> { if ( throwable != null ) { - throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + broker + ":" + brokerPort + ". Please try again.", throwable ); + throw new RuntimeException( INTERFACE_NAME + " could not disconnect from MQTT broker " + brokerAddress + ":" + brokerPort + ". Please try again.", throwable ); } else { log.info( "{} stopped.", INTERFACE_NAME ); monitoringPage.remove(); @@ -388,9 +422,9 @@ protected void reloadSettings( List updatedSettings ) { this.commonCollection.set( Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ) ); createAllCollections(); break; - case "collectionName": - String newCollectionName = this.getCurrentSettings().get( "collectionName" ).trim(); - newCollectionName = newCollectionName == null ? null : newCollectionName.trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + case "commonCollectionName": + String newCommonCollectionName = this.getCurrentSettings().get( "commonCollectionName" ).trim(); + newCommonCollectionName = newCommonCollectionName == null ? null : newCommonCollectionName.trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); boolean mode; if ( updatedSettings.contains( "commonCollection" ) ) { mode = Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ); @@ -398,19 +432,19 @@ protected void reloadSettings( List updatedSettings ) { mode = this.commonCollection.get(); } if ( mode ) { - if ( !(newCollectionName.equals( "null" ) || newCollectionName.isEmpty() || newCollectionName.isBlank()) ) { - if ( !collectionExists( newCollectionName ) ) { - createStreamCollection( this.collectionName ); + if ( !(newCommonCollectionName.equals( "null" ) || newCommonCollectionName.isEmpty() || newCommonCollectionName.isBlank()) ) { + if ( !collectionExists( newCommonCollectionName ) ) { + createStreamCollection( this.commonCollectionName ); } - this.collectionName = newCollectionName; + this.commonCollectionName = newCommonCollectionName; createAllCollections(); } else { - this.settings.put( "collectionName", this.collectionName ); + this.settings.put( "commonCollectionName", this.commonCollectionName ); throw new NullPointerException( "commonCollection is set to FALSE but no valid collection name was given! Please enter a collection name." ); } } else { - this.collectionName = newCollectionName; + this.commonCollectionName = newCommonCollectionName; } break; case "filterQuery": @@ -491,7 +525,7 @@ public void unsubscribe( String topic ) { } - void processMsg( Mqtt3Publish subMsg ) { + void processMsg( Mqtt5Publish subMsg ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); @@ -533,7 +567,7 @@ private void insert( MqttMessage mqttMessage, Transaction transaction ) { collectionToBeSaved = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); } else { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.collectionName ); + receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.commonCollectionName ); } } StreamCapture streamCapture = new StreamCapture( transaction ); @@ -542,7 +576,7 @@ private void insert( MqttMessage mqttMessage, Transaction transaction ) { // helper methods: - private static String extractPayload( Mqtt3Publish subMsg ) { + private static String extractPayload( Mqtt5Publish subMsg ) { return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); } @@ -669,9 +703,9 @@ private void createAllCollections() { } } } else { - if ( !(this.collectionName.equals( null ) || this.collectionName.equals( "" ) || this.collectionName.isBlank()) ) { - if ( !collectionExists( this.collectionName ) ) { - createStreamCollection( this.collectionName ); + if ( !(this.commonCollectionName == null || this.commonCollectionName.equals( "" ) || this.commonCollectionName.isBlank()) ) { + if ( !collectionExists( this.commonCollectionName ) ) { + createStreamCollection( this.commonCollectionName ); } } else { throw new NullPointerException( "commonCollection is set to 'true' but no valid collection name was given! Please enter a collection name." ); @@ -724,7 +758,7 @@ public String getInterfaceType() { private static class SslHelper { - private KeyFactory getKeyFactoryInstance() { + private static KeyFactory getKeyFactoryInstance() { try { return KeyFactory.getInstance( "RSA" ); } catch ( NoSuchAlgorithmException e ) { @@ -733,7 +767,7 @@ private KeyFactory getKeyFactoryInstance() { } - private X509Certificate createX509CertificateFromFile( final String certificateFileName ) { + private static X509Certificate createX509CertificateFromFile( final String certificateFileName ) { String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + certificateFileName; final File file = PolyphenyHomeDirManager.getInstance().getFileIfExists( path ); if ( !file.isFile() ) { @@ -775,37 +809,65 @@ private static KeyManagerFactory createKeyManagerFactory( String clientCertifica try { //todo: this.getUniqueName() in path String clientCrtPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; - - // load client certificate: File crtFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( clientCrtPath ); + //final X509Certificate clientCertificate = createX509CertificateFromFile("polyphenyClient.p12"); InputStream crtStream = new FileInputStream( crtFile ); + final KeyStore ks = KeyStore.getInstance( "PKCS12" ); + ks.load(crtStream,"1234".toCharArray()); + X509Certificate clientCertificate = (X509Certificate) ks.getCertificate( "alias" ); + keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(ks, "".toCharArray()); + /* + // load client certificate: + + //CertificateFactory cf = CertificateFactory.getInstance("X.509"); //Certificate clientCertificate = cf.generateCertificate( crtStream ); + crtStream.close();*/ + + // + //ks.load( null, null ); + //Certificate clientCertificate = ks.getCertificate( "alias" ); - final KeyStore ks = KeyStore.getInstance( KeyStore.getDefaultType() ); - ks.load( crtStream, null ); - Certificate clientCertificate = ks.getCertificate( "alias" ); - crtStream.close(); //load client key: String keyPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.key"; File keyFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( keyPath ); InputStream keyStream = new FileInputStream( keyFile ); + /* String key = keyStream.toString(); key = key.replace( "-----BEGIN PRIVATE KEY-----", "" ).replace( "-----END PRIVATE KEY-----", "" ); - final PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec( key.getBytes() ); - KeyFactory kf = KeyFactory.getInstance( "RSA" ); - PrivateKey privateKey = kf.generatePrivate( spec ); - // create keystore - final KeyStore keyStore = KeyStore.getInstance( KeyStore.getDefaultType() ); - keyStore.load( null, null ); + */ + + + - keyStore.setCertificateEntry( "certificate", clientCertificate ); - keyStore.setKeyEntry( "private-key", privateKey, "".toCharArray(), new Certificate[]{ clientCertificate } ); - keyManagerFactory = KeyManagerFactory.getInstance( KeyManagerFactory.getDefaultAlgorithm() ); - keyManagerFactory.init( keyStore, "".toCharArray() ); + /* + + // create private key method:------------------------------------------ + final PemReader pemReader = new PemReader(new FileReader(keyFile)); + final PemObject pemObject = pemReader.readPemObject(); + final byte[] pemContent = pemObject.getContent(); + pemReader.close(); + final PKCS8EncodedKeySpec encodedKeySpec = new PKCS8EncodedKeySpec(pemContent); + final KeyFactory keyFactory = getKeyFactoryInstance(); + final PrivateKey privateKey = keyFactory.generatePrivate(encodedKeySpec); + + //--------------------------------------------------------------------- + + + // create keystore + final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, null); + keyStore.setCertificateEntry("certificate", clientCertificate); + keyStore.setKeyEntry("private-key", privateKey, + "".toCharArray(), + new Certificate[] { clientCertificate }); + keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, "".toCharArray()); + */ } catch ( Exception e ) { throw new RuntimeException( e ); } @@ -818,13 +880,15 @@ private static TrustManagerFactory createTrustManagerFactory( final String caCer try { // load ca certificate: //todo: this.getUniqueName() in path + /* String caPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "ca.crt"; File caFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( caPath ); InputStream ca = new FileInputStream( caFile ); CertificateFactory cf = CertificateFactory.getInstance( "X.509" ); Certificate caCertificate = cf.generateCertificate( ca ); ca.close(); - +*/ + final X509Certificate caCertificate = (X509Certificate) createX509CertificateFromFile(caCertificateFileName); final KeyStore trustStore = KeyStore.getInstance( KeyStore.getDefaultType() ); trustStore.load( null, null ); trustStore.setCertificateEntry( "ca-certificate", caCertificate ); @@ -937,7 +1001,7 @@ public void update() { brokerKv.putPair( "Client state", client.getState() + "" ); brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get() + "" ); //TODO: check this after having SSL Configuration. - brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig().get() + "" ); + brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java new file mode 100644 index 0000000000..1900f8733a --- /dev/null +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -0,0 +1,139 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.iface.QueryInterface; +import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; +import org.polypheny.db.transaction.TransactionManager; + +public class MqttStreamClientTest { + static TransactionManager transactionManager; + static Map initialSettings = new HashMap<>(); + static Map changedSettings = new HashMap<>(); + @BeforeClass + public static void init() { + TestHelper testHelper = TestHelper.getInstance(); + transactionManager = testHelper.getTransactionManager(); + + } + + + @Before + public void resetSettings() { + initialSettings.clear(); + initialSettings.put( "brokerAddress", "localhost" ); + initialSettings.put( "brokerPort", "1883" ); + initialSettings.put( "commonCollectionName", "testCollection" ); + initialSettings.put( "commonCollection", "true"); + initialSettings.put( "namespace", "testNamespace"); + initialSettings.put( "namespaceType", "DOCUMENT"); + initialSettings.put( "topics", ""); + initialSettings.put( "Tsl/SslConnection", "false"); + initialSettings.put( "filterQuery", ""); + + changedSettings.clear(); + changedSettings.put( "commonCollectionName", "testCollection" ); + changedSettings.put( "commonCollection", "true"); + changedSettings.put( "namespace", "testNamespace"); + changedSettings.put( "namespaceType", "DOCUMENT"); + changedSettings.put( "topics", ""); + changedSettings.put( "filterQuery", ""); + } + + +// TODO in tEst auch prüfen ob collection name richtig ist mit _ + //TODO: Tests for save Query methode nur!! + // TODO: UI komponenten testen + + @Test + public void saveQueryTest() { + Catalog mockedCatalog = mock( Catalog.class ); + QueryInterface iface = QueryInterfaceManager.getInstance().addQueryInterface( mockedCatalog, "MqttStreamServer", "testmqtt", initialSettings ); + MqttStreamServer client = new MqttStreamServer( + transactionManager, + null, + iface.getQueryInterfaceId(), + iface.getUniqueName(), + initialSettings); + + System.out.println(client.getUniqueName()); + assertEquals( "testCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); + + changedSettings.replace( "commonCollectionName", "buttonCollection" ); + client.updateSettings( changedSettings ); + assertEquals( "buttonCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); + } + + @Test + public void reloadSettingsTopicTest() { + MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); + List updatedSettings = new ArrayList<>(); + updatedSettings.add( "topics" ); + mockPlugin.reloadSettings( updatedSettings ); + } + + public void reloadSettingsNamespaceTest() { + QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); + MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); + List updatedSettings = new ArrayList<>(); + updatedSettings.add( "namespace" ); + mockPlugin.reloadSettings( updatedSettings ); + } + + public void reloadSettingsNamespaceTypeTest() { + MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); + List updatedSettings = new ArrayList<>(); + updatedSettings.add( "namespaceType" ); + mockPlugin.reloadSettings( updatedSettings ); + } + + public void reloadSettingsNamespaceNamespaceTypeTest1() { + MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); + List updatedSettings = new ArrayList<>(); + updatedSettings.add( "namespaceType" ); + updatedSettings.add( "namespace" ); + mockPlugin.reloadSettings( updatedSettings ); + } + + public void reloadSettingsNamespaceNamespaceTypeTest2() { + MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); + List updatedSettings = new ArrayList<>(); + updatedSettings.add( "namespace" ); + updatedSettings.add( "namespaceType" ); + mockPlugin.reloadSettings( updatedSettings ); + } + + public void reloadSettingsCommonCollectionTest() {} + + public void reloadSettingsCommonCollectionNameTest() {} +} diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java deleted file mode 100644 index c398fad5e0..0000000000 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.mqtt; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.polypheny.db.TestHelper; -import org.polypheny.db.transaction.Statement; -import org.polypheny.db.transaction.Transaction; - -public class MqttTest { - @BeforeClass - public static void init() { - TestHelper.getInstance(); - - } - @Test - public void algTest() { - Transaction transaction = TestHelper.getInstance().getTransaction(); - Statement st = transaction.createStatement(); - String filterQuery = "{\"value\":11}"; - MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); - streamProcessor.processStream(); - } -} From db22a1dc20bae5e32e6eda2480c9c37e24f8f0cc Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Wed, 13 Sep 2023 19:15:42 +0200 Subject: [PATCH 087/114] MQTT version 5, create topic collection in constructor, added automatic reconnect --- .../org/polypheny/db/mqtt/MqttStreamPlugin.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index e4dc25640a..ef2023ce1c 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,8 +19,6 @@ import com.google.common.collect.ImmutableList; import com.hivemq.client.mqtt.MqttClient; -import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient; -import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish; import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient; import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.io.File; @@ -212,6 +210,15 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.createCommonCollection = true; createStreamCollection( this.commonCollectionName ); } + } else { + for( String topic : toList( settings.get( "topics" ) ) ) { + topic = topic.replace( '#', '_' ) + .replace( '+', '_' ) + .replace( '/', '_' ); + if ( !this.commonCollection.get() && !collectionExists( topic ) ) { + createStreamCollection( topic ); + } + } } String queryString = settings.get( "filterQuery" ); if ( queryString != null && !queryString.isBlank() ) { @@ -231,6 +238,7 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) + .automaticReconnectWithDefaultConfig() .sslConfig() //TODO: delete or enter password from GUI password thinghere and in method .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ) @@ -242,6 +250,7 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) + .automaticReconnectWithDefaultConfig() .buildAsync(); } @@ -494,9 +503,6 @@ public void subscribe( String topic ) { throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); } else { this.topicsMap.put( topic, new AtomicLong( 0 ) ); - if ( !this.commonCollection.get() && !collectionExists( topic ) ) { - createStreamCollection( topic ); - } } } ); //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). From 11302bae1b1551df259c65826c1476612e0a053f Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 15 Sep 2023 11:09:03 +0200 Subject: [PATCH 088/114] Debugging with Unit tests and added Qos made namespace unmodifiable, Tested: saveQuery, reload, toList, --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 44 ++-- .../db/mqtt/MqttStreamClientTest.java | 220 ++++++++++++++---- 2 files changed, 202 insertions(+), 62 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index ef2023ce1c..0d29656b65 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.hivemq.client.mqtt.MqttClient; +import com.hivemq.client.mqtt.datatypes.MqttQos; import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient; import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.io.File; @@ -42,6 +43,7 @@ import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import javax.net.ssl.KeyManagerFactory; @@ -136,8 +138,8 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingInteger( "brokerPort", false, true, false, null ), new QueryInterfaceSettingString( "namespace", false, true, true, null ), // "RELATIONAL", "GRAPH" types are not supported yet. - new QueryInterfaceSettingList( "namespaceType", false, true, true, - new ArrayList<>( List.of( "DOCUMENT", "RELATIONAL", "GRAPH" ) ) ), + new QueryInterfaceSettingList( "namespaceType", false, true, false, + new ArrayList<>( List.of( "DOCUMENT") ) ), new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "commonCollectionName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), @@ -170,7 +172,7 @@ public static class MqttStreamServer extends QueryInterface { private String commonCollectionName; private final long databaseId; private final int userId; - final boolean ssl; // todo this. + private final boolean ssl; // todo this. boolean createCommonCollection = false; private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -210,7 +212,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.createCommonCollection = true; createStreamCollection( this.commonCollectionName ); } - } else { + } else if ( settings.get( "topics" ) != null ) { for( String topic : toList( settings.get( "topics" ) ) ) { topic = topic.replace( '#', '_' ) .replace( '+', '_' ) @@ -238,7 +240,6 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) - .automaticReconnectWithDefaultConfig() .sslConfig() //TODO: delete or enter password from GUI password thinghere and in method .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ) @@ -250,13 +251,12 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) - .automaticReconnectWithDefaultConfig() .buildAsync(); } client.connectWith().send().whenComplete( ( connAck, throwable ) -> { if ( throwable != null ) { - throw new RuntimeException( "Connection to broker could not be established. Please delete and recreate the Plug-In." + throwable ); + throw new RuntimeException( "Connection to broker could not be established. Try to reconnect with the 'Reconnect' button" + throwable ); } else { log.info( "{} started and is listening to broker on {}:{}", INTERFACE_NAME, brokerAddress, brokerPort ); subscribe( toList( this.settings.get( "topics" ) ) ); @@ -500,13 +500,11 @@ public void subscribe( String topic ) { this.settings.put( "topics", topicsString ); } log.info( "not successful: {}", topic ); - throw new RuntimeException( "Subscription was not successful. Please try again.", throwable ); + throw new RuntimeException( String.format( "Subscription was not successful for topic \"%s\" . Please try again.", topic), throwable ); } else { this.topicsMap.put( topic, new AtomicLong( 0 ) ); } } ); - //info: no notify() here, because otherwise only the first topic will be subscribed from the method subscribeToAll(). - } @@ -598,7 +596,11 @@ private void saveQueriesInMap( String queries ) { while ( !queries.isBlank() ) { int index = 0; String topic = queries.substring( 0, queries.indexOf( ":" ) ); - queries = queries.replace( topic + ":", "" ); + queries = queries.substring( queries.indexOf( ":" ) + 1 ); + if ( topic.startsWith( "," ) || topic.startsWith( " ," ) ) { + topic = topic.replaceFirst( ",", "" ).trim(); + } + while ( !queries.isBlank() ) { char c = queries.charAt( index ); if ( c == '{' ) { @@ -607,9 +609,11 @@ private void saveQueriesInMap( String queries ) { } else if ( c == '}' ) { if ( brackets.pop().equals( '{' ) ) { if ( brackets.isEmpty() ) { - query = queries.substring( 0, index + 1 ); - if ( this.filterMap.containsKey( topic ) && !this.filterMap.get( topic ).equals( queries ) ) { - this.filterMap.replace( topic, query ); + query = queries.substring( 0, index + 1 ).trim(); + if ( this.filterMap.containsKey( topic ) ) { + if ( !this.filterMap.get( topic ).equals( query ) ) { + this.filterMap.replace( topic, query ); + } } else { this.filterMap.put( topic, query ); } @@ -647,15 +651,17 @@ private String getWildcardTopic( String topic ) { /** * separates a string by commas and inserts the separated parts to a list. * - * @param string + * @param string List of Strings seperated by comma without brackets as a String (entry form UI) * @return List of seperated string values */ public List toList( String string ) { List list = new ArrayList<>( List.of( string.split( "," ) ) ); for ( int i = 0; i < list.size(); i++ ) { String topic = list.get( i ).trim(); - if ( !topic.isBlank() ) { + if ( !topic.isBlank() || !topic.isEmpty() ) { list.set( i, topic ); + } else { + list.remove( i ); } } return list; @@ -952,7 +958,11 @@ public MonitoringPage() { msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { String end = "Msg was published!"; try { - client.publishWith().topic( parameters.get( "topic" ) ).payload( parameters.get( "msg" ).getBytes() ).send(); + client.publishWith() + .topic( parameters.get( "topic" ) ) + .payload( parameters.get( "msg" ).getBytes() ) + .qos( MqttQos.AT_LEAST_ONCE ) + .send(); } catch ( IllegalArgumentException e ) { throw new RuntimeException( e ); } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index 1900f8733a..7a1544e6a6 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -27,21 +27,28 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.verification.VerificationMode; import org.polypheny.db.TestHelper; import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; +import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; public class MqttStreamClientTest { static TransactionManager transactionManager; + static Transaction transaction; static Map initialSettings = new HashMap<>(); static Map changedSettings = new HashMap<>(); + + MqttStreamServer client; @BeforeClass public static void init() { TestHelper testHelper = TestHelper.getInstance(); transactionManager = testHelper.getTransactionManager(); + transaction = testHelper.getTransaction(); } @@ -59,11 +66,21 @@ public void resetSettings() { initialSettings.put( "Tsl/SslConnection", "false"); initialSettings.put( "filterQuery", ""); + QueryInterface iface = QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); + + client = new MqttStreamServer( + transactionManager, + null, + iface.getQueryInterfaceId(), + iface.getUniqueName(), + initialSettings); + + changedSettings.clear(); changedSettings.put( "commonCollectionName", "testCollection" ); changedSettings.put( "commonCollection", "true"); changedSettings.put( "namespace", "testNamespace"); - changedSettings.put( "namespaceType", "DOCUMENT"); + //changedSettings.put( "namespaceType", "DOCUMENT"); changedSettings.put( "topics", ""); changedSettings.put( "filterQuery", ""); } @@ -74,66 +91,179 @@ public void resetSettings() { // TODO: UI komponenten testen @Test - public void saveQueryTest() { - Catalog mockedCatalog = mock( Catalog.class ); - QueryInterface iface = QueryInterfaceManager.getInstance().addQueryInterface( mockedCatalog, "MqttStreamServer", "testmqtt", initialSettings ); - MqttStreamServer client = new MqttStreamServer( - transactionManager, - null, - iface.getQueryInterfaceId(), - iface.getUniqueName(), - initialSettings); + public void saveQueryEmptyStringTest() { + changedSettings.replace( "filterQuery", " " ); + client.updateSettings( changedSettings ); + Map expected = new HashMap<>(0); + assertEquals( expected, client.getFilterMap() ); + } - System.out.println(client.getUniqueName()); - assertEquals( "testCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); - changedSettings.replace( "commonCollectionName", "buttonCollection" ); + @Test + public void saveSimpleQueryTest() { + changedSettings.replace( "filterQuery", "topic1:{key1:value1}" ); client.updateSettings( changedSettings ); - assertEquals( "buttonCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); + Map expected = new HashMap<>(1); + expected.put( "topic1", "{key1:value1}" ); + assertEquals( expected, client.getFilterMap() ); + } + + + @Test + public void saveQueryWithArrayTest() { + changedSettings.replace( "filterQuery", "topic1:{key1:[1, 2, 3]}" ); + client.updateSettings( changedSettings ); + Map expected = new HashMap<>(1); + expected.put( "topic1", "{key1:[1, 2, 3]}" ); + assertEquals( expected, client.getFilterMap() ); + } + + + @Test + public void saveTwoSimpleQueryTest() { + changedSettings.replace( "filterQuery", "topic1:{key1:value1}, topic2:{key2:value2}" ); + client.updateSettings( changedSettings ); + Map expected = new HashMap<>(2); + expected.put( "topic1", "{key1:value1}" ); + expected.put( "topic2", "{key2:value2}" ); + assertEquals( expected, client.getFilterMap() ); } + + + @Test + public void saveNestedQueryTest() { + changedSettings.replace( "filterQuery", "topic1:{key1:{$lt:3}}, topic2:{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); + client.updateSettings( changedSettings ); + Map expected = new HashMap<>(2); + expected.put( "topic1", "{key1:{$lt:3}}" ); + expected.put( "topic2", "{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); + assertEquals( expected, client.getFilterMap() ); + } + + + + @Test + public void toListEmptyTest() { + List result = client.toList( "" ); + List expected = new ArrayList<>(); + assertEquals( expected, result ); + } + + @Test + public void toListSpaceTest() { + List result = client.toList( " " ); + List expected = new ArrayList<>(); + assertEquals( expected, result ); + } + + + @Test + public void toListWithContentTest() { + List result = client.toList( "1, 2 " ); + List expected = new ArrayList<>(); + expected.add( "1" ); + expected.add( "2" ); + assertEquals( expected, result ); + } + @Test public void reloadSettingsTopicTest() { - MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); - List updatedSettings = new ArrayList<>(); - updatedSettings.add( "topics" ); - mockPlugin.reloadSettings( updatedSettings ); + //TODO with broker + //TODO: with wildcards } - public void reloadSettingsNamespaceTest() { - QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); - MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); - List updatedSettings = new ArrayList<>(); - updatedSettings.add( "namespace" ); - mockPlugin.reloadSettings( updatedSettings ); + @Test + public void reloadSettingsNewNamespaceTest() { + // change to new namespace: + changedSettings.replace( "namespace", "namespace2" ); + client.updateSettings( changedSettings ); + assertEquals( "namespace2", client.getNamespaceName() ); + } + + @Test + public void reloadSettingsExistingNamespaceTest() { + // change to existing namespace: + changedSettings.replace( "namespace", "testNamespace" ); + client.updateSettings( changedSettings ); + assertEquals( "testNamespace", client.getNamespaceName() ); + } + + @Test(expected = RuntimeException.class) + public void reloadSettingsWrongTypeNamespaceTest() { + // change to existing namespace other type: + changedSettings.replace( "namespace", "public" ); + client.updateSettings( changedSettings ); + assertEquals( "testNamespace", client.getNamespaceName() ); + } + + + @Test + public void reloadSettingsCommonCollectionToFalseTest() { + changedSettings.replace( "commonCollection", "false" ); + client.updateSettings( changedSettings ); + assertFalse( client.getCommonCollection().get() ); + } + + @Test + public void reloadSettingsCommonCollectionToTrueTest() { + changedSettings.replace( "commonCollection", "true" ); + client.updateSettings( changedSettings ); + assertTrue( client.getCommonCollection().get() ); + } + + @Test + public void reloadSettingsNewCommonCollectionNameTest() { + changedSettings.replace( "commonCollectionName", "buttonCollection" ); + client.updateSettings( changedSettings ); + assertEquals( "buttonCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); } - public void reloadSettingsNamespaceTypeTest() { - MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); - List updatedSettings = new ArrayList<>(); - updatedSettings.add( "namespaceType" ); - mockPlugin.reloadSettings( updatedSettings ); + + @Test + public void reloadSettingsExistingCommonCollectionNameTest() { + changedSettings.replace( "commonCollectionName", "testCollection" ); + client.updateSettings( changedSettings ); + assertEquals( "testCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); } - public void reloadSettingsNamespaceNamespaceTypeTest1() { - MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); - List updatedSettings = new ArrayList<>(); - updatedSettings.add( "namespaceType" ); - updatedSettings.add( "namespace" ); - mockPlugin.reloadSettings( updatedSettings ); + + @Test + public void reloadSettingsCommonCollectionAndCommonCollectionNameTest() { + // testing special case: commonCollection changed from false to true + commonCollectionName changes + changedSettings.replace( "commonCollection", "false" ); + client.updateSettings( changedSettings ); + + + changedSettings.replace( "commonCollectionName", "buttonCollection" ); + changedSettings.replace( "commonCollection", "true" ); + client.updateSettings( changedSettings ); + assertEquals( "buttonCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); } - public void reloadSettingsNamespaceNamespaceTypeTest2() { - MqttStreamServer mockPlugin = mock( MqttStreamPlugin.MqttStreamServer.class); - List updatedSettings = new ArrayList<>(); - updatedSettings.add( "namespace" ); - updatedSettings.add( "namespaceType" ); - mockPlugin.reloadSettings( updatedSettings ); + @Test(expected = NullPointerException.class) + public void reloadSettingsCommonCollectionAndCommonCollectionNameTest2() { + // testing special case: commonCollection changed from false to true + commonCollectionName changes + changedSettings.replace( "commonCollection", "false" ); + client.updateSettings( changedSettings ); + + + changedSettings.replace( "commonCollectionName", " " ); + changedSettings.replace( "commonCollection", "true" ); + client.updateSettings( changedSettings ); + assertEquals( "testCollection", client.getCommonCollectionName() ); + assertTrue( client.getCommonCollection().get() ); } - public void reloadSettingsCommonCollectionTest() {} - public void reloadSettingsCommonCollectionNameTest() {} + + + + @Test + public void processMsgTest() { +//TODO + } } From 1ef13a8004928641c852e2e23a5bbbc26cadd25b Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 15 Sep 2023 20:49:55 +0200 Subject: [PATCH 089/114] testing and debugging --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 224 ++++++++---------- .../db/mqtt/MqttStreamClientTest.java | 93 ++++---- 2 files changed, 154 insertions(+), 163 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 0d29656b65..01ea242b1a 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -24,17 +24,13 @@ import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.io.File; import java.io.FileInputStream; -import java.io.FileReader; import java.io.InputStream; import java.nio.charset.Charset; import java.security.KeyFactory; import java.security.KeyStore; import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; -import java.security.spec.PKCS8EncodedKeySpec; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -43,16 +39,12 @@ import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.util.io.pem.PemObject; -import org.bouncycastle.util.io.pem.PemReader; import org.pf4j.Extension; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; @@ -80,9 +72,6 @@ import org.polypheny.db.information.InformationManager; import org.polypheny.db.information.InformationPage; import org.polypheny.db.information.InformationTable; -import org.polypheny.db.plan.AlgOptCluster; -import org.polypheny.db.rex.RexBuilder; -import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; @@ -139,7 +128,7 @@ public static class MqttStreamServer extends QueryInterface { new QueryInterfaceSettingString( "namespace", false, true, true, null ), // "RELATIONAL", "GRAPH" types are not supported yet. new QueryInterfaceSettingList( "namespaceType", false, true, false, - new ArrayList<>( List.of( "DOCUMENT") ) ), + new ArrayList<>( List.of( "DOCUMENT" ) ) ), new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "commonCollectionName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), @@ -201,10 +190,10 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au this.commonCollection = new AtomicBoolean( Boolean.parseBoolean( settings.get( "commonCollection" ) ) ); this.commonCollectionName = settings.get( "commonCollectionName" ) == null ? settings.get( "commonCollectionName" ) : settings.get( "commonCollectionName" ) - .trim() - .replace( '#', '_' ) - .replace( '+', '_' ) - .replace( '/', '_' ); + .trim() + .replace( '#', '_' ) + .replace( '+', '_' ) + .replace( '/', '_' ); if ( this.commonCollection.get() ) { if ( this.commonCollectionName == null || this.commonCollectionName.isEmpty() || this.commonCollectionName.isBlank() ) { throw new NullPointerException( "commonCollection is set to true but no valid collection name was given! Please enter a collection name." ); @@ -213,7 +202,7 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au createStreamCollection( this.commonCollectionName ); } } else if ( settings.get( "topics" ) != null ) { - for( String topic : toList( settings.get( "topics" ) ) ) { + for ( String topic : toList( settings.get( "topics" ) ) ) { topic = topic.replace( '#', '_' ) .replace( '+', '_' ) .replace( '/', '_' ); @@ -235,7 +224,6 @@ public MqttStreamServer( TransactionManager transactionManager, Authenticator au @Override public void run() { if ( ssl ) { - this.client = MqttClient.builder().useMqttVersion5() .identifier( getUniqueName() ) .serverHost( brokerAddress ) @@ -467,7 +455,7 @@ protected void reloadSettings( List updatedSettings ) { } - void subscribe( List newTopics ) { + protected void subscribe( List newTopics ) { for ( String t : newTopics ) { subscribe( t ); } @@ -479,43 +467,44 @@ void subscribe( List newTopics ) { * * @param topic the topic the client should subscribe to. */ - public void subscribe( String topic ) { - client.subscribeWith().topicFilter( topic ).callback( subMsg -> { - processMsg( subMsg ); - } ).send().whenComplete( ( subAck, throwable ) -> { - if ( throwable != null ) { - //TODO: change settings correctly: - List topicsList = toList( this.getCurrentSettings().get( "topics" ) ); - StringBuilder stringBuilder = new StringBuilder(); - for ( String t : topicsList ) { - if ( !t.equals( topic ) ) { - stringBuilder.append( t ).append( "," ); + protected void subscribe( String topic ) { + client.subscribeWith().topicFilter( topic ) + .callback( this::processMsg ) + .send() + .whenComplete( ( subAck, throwable ) -> { + if ( throwable != null ) { + //TODO: change settings correctly: + List topicsList = toList( this.getCurrentSettings().get( "topics" ) ); + StringBuilder stringBuilder = new StringBuilder(); + for ( String t : topicsList ) { + if ( !t.equals( topic ) ) { + stringBuilder.append( t ).append( "," ); + } + } + String topicsString = stringBuilder.toString(); + if ( topicsString != null && !topicsString.isBlank() ) { + topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); + } + synchronized ( settingsLock ) { + this.settings.put( "topics", topicsString ); + } + log.info( "not successful: {}", topic ); + throw new RuntimeException( String.format( "Subscription was not successful for topic \"%s\" . Please try again.", topic ), throwable ); + } else { + this.topicsMap.put( topic, new AtomicLong( 0 ) ); } - } - String topicsString = stringBuilder.toString(); - if ( topicsString != null && !topicsString.isBlank() ) { - topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); - } - synchronized ( settingsLock ) { - this.settings.put( "topics", topicsString ); - } - log.info( "not successful: {}", topic ); - throw new RuntimeException( String.format( "Subscription was not successful for topic \"%s\" . Please try again.", topic), throwable ); - } else { - this.topicsMap.put( topic, new AtomicLong( 0 ) ); - } - } ); + } ); } - public void unsubscribe( List topics ) { + protected void unsubscribe( List topics ) { for ( String t : topics ) { unsubscribe( t ); } } - public void unsubscribe( String topic ) { + protected void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { synchronized ( settingsLock ) { @@ -529,12 +518,62 @@ public void unsubscribe( String topic ) { } - void processMsg( Mqtt5Publish subMsg ) { + /** + * format of queries comma seperated: :, :, ... + * + * @param queries + */ + private void saveQueriesInMap( String queries ) { + Stack brackets = new Stack<>(); + String query; + while ( !queries.isBlank() ) { + int index = 0; + String topic = queries.substring( 0, queries.indexOf( ":" ) ); + queries = queries.substring( queries.indexOf( ":" ) + 1 ); + if ( topic.startsWith( "," ) || topic.startsWith( " ," ) ) { + topic = topic.replaceFirst( ",", "" ).trim(); + } + + while ( !queries.isBlank() ) { + char c = queries.charAt( index ); + if ( c == '{' ) { + brackets.push( c ); + index++; + } else if ( c == '}' ) { + if ( brackets.pop().equals( '{' ) ) { + if ( brackets.isEmpty() ) { + query = queries.substring( 0, index + 1 ).trim(); + if ( this.filterMap.containsKey( topic ) ) { + if ( !this.filterMap.get( topic ).equals( query ) ) { + this.filterMap.replace( topic, query ); + } + } else { + this.filterMap.put( topic, query ); + } + queries = queries.substring( index + 1 ); + break; + } + } else { + throw new RuntimeException( String.format( "The brackets in the query to the topic %s are not set correctly!", topic ) ); + } + } + if ( index < queries.toCharArray().length ) { + index++; + } + } + } + } + + + protected void processMsg( Mqtt5Publish subMsg ) { Transaction transaction = getTransaction(); Statement statement = transaction.createStatement(); String topic = subMsg.getTopic().toString(); String message = extractPayload( subMsg ); + MqttMessage mqttMessage = new MqttMessage( message, topic ); + addMessageToQueue( topic, message ); + String wildcardTopic = ""; if ( !topicsMap.containsKey( topic ) ) { wildcardTopic = getWildcardTopic( topic ); @@ -542,8 +581,7 @@ void processMsg( Mqtt5Publish subMsg ) { } else { topicsMap.get( topic ).incrementAndGet(); } - MqttMessage mqttMessage = new MqttMessage( message, topic ); - addMessageToQueue( topic, message ); + if ( this.filterMap.containsKey( topic ) ) { String filterQuery = this.filterMap.get( topic ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); @@ -579,59 +617,11 @@ private void insert( MqttMessage mqttMessage, Transaction transaction ) { } - // helper methods: private static String extractPayload( Mqtt5Publish subMsg ) { return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); } - /** - * format of queries comma seperated: :, :, ... - * - * @param queries - */ - private void saveQueriesInMap( String queries ) { - Stack brackets = new Stack<>(); - String query; - while ( !queries.isBlank() ) { - int index = 0; - String topic = queries.substring( 0, queries.indexOf( ":" ) ); - queries = queries.substring( queries.indexOf( ":" ) + 1 ); - if ( topic.startsWith( "," ) || topic.startsWith( " ," ) ) { - topic = topic.replaceFirst( ",", "" ).trim(); - } - - while ( !queries.isBlank() ) { - char c = queries.charAt( index ); - if ( c == '{' ) { - brackets.push( c ); - index++; - } else if ( c == '}' ) { - if ( brackets.pop().equals( '{' ) ) { - if ( brackets.isEmpty() ) { - query = queries.substring( 0, index + 1 ).trim(); - if ( this.filterMap.containsKey( topic ) ) { - if ( !this.filterMap.get( topic ).equals( query ) ) { - this.filterMap.replace( topic, query ); - } - } else { - this.filterMap.put( topic, query ); - } - queries = queries.substring( index + 1 ); - break; - } - } else { - throw new RuntimeException( String.format( "The brackets in the query to the topic %s are not set correctly!", topic ) ); - } - } - if ( index < queries.toCharArray().length ) { - index++; - } - } - } - } - - private String getWildcardTopic( String topic ) { for ( String t : topicsMap.keySet() ) { //multilevel wildcard @@ -648,13 +638,23 @@ private String getWildcardTopic( String topic ) { } + private void addMessageToQueue( String topic, String message ) { + if ( this.messageQueue.size() >= 20 ) { + this.messageQueue.poll(); + this.messageQueue.add( new String[]{ topic, message } ); + } else { + this.messageQueue.add( new String[]{ topic, message } ); + } + } + + /** * separates a string by commas and inserts the separated parts to a list. * * @param string List of Strings seperated by comma without brackets as a String (entry form UI) * @return List of seperated string values */ - public List toList( String string ) { + protected List toList( String string ) { List list = new ArrayList<>( List.of( string.split( "," ) ) ); for ( int i = 0; i < list.size(); i++ ) { String topic = list.get( i ).trim(); @@ -728,16 +728,6 @@ private void createAllCollections() { } - private void addMessageToQueue( String topic, String message ) { - if ( this.messageQueue.size() >= 20 ) { - this.messageQueue.poll(); - this.messageQueue.add( new String[]{ topic, message } ); - } else { - this.messageQueue.add( new String[]{ topic, message } ); - } - } - - private Transaction getTransaction() { try { return transactionManager.startTransaction( this.userId, this.databaseId, false, "MQTT Stream" ); @@ -747,15 +737,6 @@ private Transaction getTransaction() { } - private AlgBuilder getAlgBuilder() { - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - final RexBuilder rexBuilder = new RexBuilder( statement.getTransaction().getTypeFactory() ); - final AlgOptCluster cluster = AlgOptCluster.create( statement.getQueryProcessor().getPlanner(), rexBuilder ); - return AlgBuilder.create( statement, cluster ); - } - - @Override public void languageChange() { @@ -825,10 +806,10 @@ private static KeyManagerFactory createKeyManagerFactory( String clientCertifica //final X509Certificate clientCertificate = createX509CertificateFromFile("polyphenyClient.p12"); InputStream crtStream = new FileInputStream( crtFile ); final KeyStore ks = KeyStore.getInstance( "PKCS12" ); - ks.load(crtStream,"1234".toCharArray()); + ks.load( crtStream, "1234".toCharArray() ); X509Certificate clientCertificate = (X509Certificate) ks.getCertificate( "alias" ); - keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(ks, "".toCharArray()); + keyManagerFactory = KeyManagerFactory.getInstance( KeyManagerFactory.getDefaultAlgorithm() ); + keyManagerFactory.init( ks, "".toCharArray() ); /* // load client certificate: @@ -841,7 +822,6 @@ private static KeyManagerFactory createKeyManagerFactory( String clientCertifica //ks.load( null, null ); //Certificate clientCertificate = ks.getCertificate( "alias" ); - //load client key: String keyPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.key"; File keyFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( keyPath ); @@ -900,7 +880,7 @@ private static TrustManagerFactory createTrustManagerFactory( final String caCer Certificate caCertificate = cf.generateCertificate( ca ); ca.close(); */ - final X509Certificate caCertificate = (X509Certificate) createX509CertificateFromFile(caCertificateFileName); + final X509Certificate caCertificate = (X509Certificate) createX509CertificateFromFile( caCertificateFileName ); final KeyStore trustStore = KeyStore.getInstance( KeyStore.getDefaultType() ); trustStore.load( null, null ); trustStore.setCertificateEntry( "ca-certificate", caCertificate ); diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index 7a1544e6a6..618fec7a3c 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -16,10 +16,17 @@ package org.polypheny.db.mqtt; -import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.hivemq.client.internal.mqtt.datatypes.MqttTopicImpl; +import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder; +import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder.Default; +import com.hivemq.client.internal.mqtt.message.publish.MqttPublish; +import com.hivemq.client.mqtt.datatypes.MqttQos; +import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -27,10 +34,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.verification.VerificationMode; import org.polypheny.db.TestHelper; -import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; @@ -38,12 +42,15 @@ import org.polypheny.db.transaction.TransactionManager; public class MqttStreamClientTest { + static TransactionManager transactionManager; static Transaction transaction; - static Map initialSettings = new HashMap<>(); - static Map changedSettings = new HashMap<>(); + static Map initialSettings = new HashMap<>(); + static Map changedSettings = new HashMap<>(); MqttStreamServer client; + + @BeforeClass public static void init() { TestHelper testHelper = TestHelper.getInstance(); @@ -59,12 +66,12 @@ public void resetSettings() { initialSettings.put( "brokerAddress", "localhost" ); initialSettings.put( "brokerPort", "1883" ); initialSettings.put( "commonCollectionName", "testCollection" ); - initialSettings.put( "commonCollection", "true"); - initialSettings.put( "namespace", "testNamespace"); - initialSettings.put( "namespaceType", "DOCUMENT"); - initialSettings.put( "topics", ""); - initialSettings.put( "Tsl/SslConnection", "false"); - initialSettings.put( "filterQuery", ""); + initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "namespace", "testNamespace" ); + initialSettings.put( "namespaceType", "DOCUMENT" ); + initialSettings.put( "topics", "" ); + initialSettings.put( "Tsl/SslConnection", "false" ); + initialSettings.put( "filterQuery", "" ); QueryInterface iface = QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); @@ -73,28 +80,28 @@ public void resetSettings() { null, iface.getQueryInterfaceId(), iface.getUniqueName(), - initialSettings); - + initialSettings ); changedSettings.clear(); changedSettings.put( "commonCollectionName", "testCollection" ); - changedSettings.put( "commonCollection", "true"); - changedSettings.put( "namespace", "testNamespace"); + changedSettings.put( "commonCollection", "true" ); + changedSettings.put( "namespace", "testNamespace" ); //changedSettings.put( "namespaceType", "DOCUMENT"); - changedSettings.put( "topics", ""); - changedSettings.put( "filterQuery", ""); + changedSettings.put( "topics", "" ); + changedSettings.put( "filterQuery", "" ); } -// TODO in tEst auch prüfen ob collection name richtig ist mit _ - //TODO: Tests for save Query methode nur!! - // TODO: UI komponenten testen + @Test + public void UITest() { // TODO: UI komponenten testen + } + @Test public void saveQueryEmptyStringTest() { changedSettings.replace( "filterQuery", " " ); client.updateSettings( changedSettings ); - Map expected = new HashMap<>(0); + Map expected = new HashMap<>( 0 ); assertEquals( expected, client.getFilterMap() ); } @@ -103,7 +110,7 @@ public void saveQueryEmptyStringTest() { public void saveSimpleQueryTest() { changedSettings.replace( "filterQuery", "topic1:{key1:value1}" ); client.updateSettings( changedSettings ); - Map expected = new HashMap<>(1); + Map expected = new HashMap<>( 1 ); expected.put( "topic1", "{key1:value1}" ); assertEquals( expected, client.getFilterMap() ); } @@ -113,7 +120,7 @@ public void saveSimpleQueryTest() { public void saveQueryWithArrayTest() { changedSettings.replace( "filterQuery", "topic1:{key1:[1, 2, 3]}" ); client.updateSettings( changedSettings ); - Map expected = new HashMap<>(1); + Map expected = new HashMap<>( 1 ); expected.put( "topic1", "{key1:[1, 2, 3]}" ); assertEquals( expected, client.getFilterMap() ); } @@ -123,7 +130,7 @@ public void saveQueryWithArrayTest() { public void saveTwoSimpleQueryTest() { changedSettings.replace( "filterQuery", "topic1:{key1:value1}, topic2:{key2:value2}" ); client.updateSettings( changedSettings ); - Map expected = new HashMap<>(2); + Map expected = new HashMap<>( 2 ); expected.put( "topic1", "{key1:value1}" ); expected.put( "topic2", "{key2:value2}" ); assertEquals( expected, client.getFilterMap() ); @@ -134,24 +141,24 @@ public void saveTwoSimpleQueryTest() { public void saveNestedQueryTest() { changedSettings.replace( "filterQuery", "topic1:{key1:{$lt:3}}, topic2:{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); client.updateSettings( changedSettings ); - Map expected = new HashMap<>(2); + Map expected = new HashMap<>( 2 ); expected.put( "topic1", "{key1:{$lt:3}}" ); expected.put( "topic2", "{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); assertEquals( expected, client.getFilterMap() ); } - @Test public void toListEmptyTest() { - List result = client.toList( "" ); + List result = client.toList( "" ); List expected = new ArrayList<>(); assertEquals( expected, result ); } + @Test public void toListSpaceTest() { - List result = client.toList( " " ); + List result = client.toList( " " ); List expected = new ArrayList<>(); assertEquals( expected, result ); } @@ -159,20 +166,21 @@ public void toListSpaceTest() { @Test public void toListWithContentTest() { - List result = client.toList( "1, 2 " ); + List result = client.toList( "1, 2 " ); List expected = new ArrayList<>(); expected.add( "1" ); expected.add( "2" ); assertEquals( expected, result ); } - + @Test public void reloadSettingsTopicTest() { //TODO with broker //TODO: with wildcards } + @Test public void reloadSettingsNewNamespaceTest() { // change to new namespace: @@ -181,6 +189,7 @@ public void reloadSettingsNewNamespaceTest() { assertEquals( "namespace2", client.getNamespaceName() ); } + @Test public void reloadSettingsExistingNamespaceTest() { // change to existing namespace: @@ -189,11 +198,12 @@ public void reloadSettingsExistingNamespaceTest() { assertEquals( "testNamespace", client.getNamespaceName() ); } + @Test(expected = RuntimeException.class) public void reloadSettingsWrongTypeNamespaceTest() { // change to existing namespace other type: changedSettings.replace( "namespace", "public" ); - client.updateSettings( changedSettings ); + client.updateSettings( changedSettings ); assertEquals( "testNamespace", client.getNamespaceName() ); } @@ -205,6 +215,7 @@ public void reloadSettingsCommonCollectionToFalseTest() { assertFalse( client.getCommonCollection().get() ); } + @Test public void reloadSettingsCommonCollectionToTrueTest() { changedSettings.replace( "commonCollection", "true" ); @@ -212,6 +223,7 @@ public void reloadSettingsCommonCollectionToTrueTest() { assertTrue( client.getCommonCollection().get() ); } + @Test public void reloadSettingsNewCommonCollectionNameTest() { changedSettings.replace( "commonCollectionName", "buttonCollection" ); @@ -236,7 +248,6 @@ public void reloadSettingsCommonCollectionAndCommonCollectionNameTest() { changedSettings.replace( "commonCollection", "false" ); client.updateSettings( changedSettings ); - changedSettings.replace( "commonCollectionName", "buttonCollection" ); changedSettings.replace( "commonCollection", "true" ); client.updateSettings( changedSettings ); @@ -244,13 +255,13 @@ public void reloadSettingsCommonCollectionAndCommonCollectionNameTest() { assertTrue( client.getCommonCollection().get() ); } + @Test(expected = NullPointerException.class) public void reloadSettingsCommonCollectionAndCommonCollectionNameTest2() { // testing special case: commonCollection changed from false to true + commonCollectionName changes changedSettings.replace( "commonCollection", "false" ); client.updateSettings( changedSettings ); - changedSettings.replace( "commonCollectionName", " " ); changedSettings.replace( "commonCollection", "true" ); client.updateSettings( changedSettings ); @@ -259,11 +270,11 @@ public void reloadSettingsCommonCollectionAndCommonCollectionNameTest2() { } - - - @Test public void processMsgTest() { -//TODO + MqttUserPropertiesImplBuilder.Default defaultProperties = new Default(); + Mqtt5Publish message = new MqttPublish( MqttTopicImpl.of( "topic1" ), ByteBuffer.wrap( "payload".getBytes() ), MqttQos.AT_LEAST_ONCE, false, 10, null, null, null, null, defaultProperties.build(), null ); + client.processMsg( message ); } + } From ce068fc9db509b110f40d48b46142b0fe9d650e0 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 15 Sep 2023 20:50:20 +0200 Subject: [PATCH 090/114] minor change at prefix for query --- .../java/org/polypheny/db/mqtt/MqttStreamProcessor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index 6c3cf84daa..81b0258755 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -23,7 +23,6 @@ import org.bson.BsonDocument; import org.bson.BsonDouble; import org.bson.BsonString; -import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.logical.document.LogicalDocumentValues; @@ -31,13 +30,11 @@ import org.polypheny.db.languages.mql.MqlFind; import org.polypheny.db.languages.mql2alg.MqlToAlgConverter; import org.polypheny.db.plan.AlgOptCluster; -import org.polypheny.db.prepare.Context; import org.polypheny.db.prepare.PolyphenyDbCatalogReader; import org.polypheny.db.processing.Processor; import org.polypheny.db.stream.StreamProcessorImpl; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; -import org.polypheny.db.transaction.TransactionException; @Slf4j public class MqttStreamProcessor extends StreamProcessorImpl { @@ -56,7 +53,7 @@ public MqttStreamProcessor( MqttMessage mqttMessage, String filterQuery, Stateme //TODO: in Tutorial schreiben, was allgemein für Strings gilt public boolean applyFilter() { AlgRoot root = processMqlQuery(); - List> res = executeAndTransformPolyAlg( root, statement); + List> res = executeAndTransformPolyAlg( root, statement ); log.info( res.toString() ); return res.size() != 0; } @@ -69,7 +66,7 @@ private AlgRoot processMqlQuery() { final AlgOptCluster cluster = AlgOptCluster.createDocument( statement.getQueryProcessor().getPlanner(), algBuilder.getRexBuilder() ); MqlToAlgConverter mqlConverter = new MqlToAlgConverter( mqlProcessor, catalogReader, cluster ); - MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "null", this.filterQuery ) ).get( 0 ); + MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "collection", this.filterQuery ) ).get( 0 ); String msg = getStream(); BsonDocument msgDoc; AlgNode input; @@ -92,4 +89,5 @@ private AlgRoot processMqlQuery() { input = LogicalDocumentValues.create( cluster, ImmutableList.of( msgDoc ) ); return mqlConverter.convert( find, input ); } + } \ No newline at end of file From e3042b417e4af9380606e71b24681d04df25ce22 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 24 Sep 2023 21:59:30 +0200 Subject: [PATCH 091/114] last time pushing multiple changes at once!! --- ...Message.java => FilteringMqttMessage.java} | 16 +++- .../polypheny/db/mqtt/MqttStreamPlugin.java | 54 +++++++------ .../db/mqtt/MqttStreamProcessor.java | 11 ++- ...ttMessage.java => StoringMqttMessage.java} | 27 +++---- .../org/polypheny/db/mqtt/StreamCapture.java | 81 +++++++++++++++++-- .../db/mqtt/MqttStreamClientTest.java | 16 ++-- .../db/mqtt/MqttStreamProcessorTest.java | 68 +++++++++------- .../polypheny/db/mqtt/StreamCaptureTest.java | 60 ++++++++++++++ 8 files changed, 240 insertions(+), 93 deletions(-) rename plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/{PublishingMqttMessage.java => FilteringMqttMessage.java} (63%) rename plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/{ReceivedMqttMessage.java => StoringMqttMessage.java} (71%) create mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java similarity index 63% rename from plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java rename to plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java index e35c47d97a..f8c9b0c8ad 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/PublishingMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java @@ -16,8 +16,20 @@ package org.polypheny.db.mqtt; -public class PublishingMqttMessage { +import lombok.Getter; - MqttMessage msg; +public class FilteringMqttMessage { + private MqttMessage mqttMessage; + @Getter + private String query; + public FilteringMqttMessage( MqttMessage mqttMessage, String query ) { + this.mqttMessage = mqttMessage; + this.query = query; + } + + + public String getMessage() { + return mqttMessage.getMessage(); + } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 01ea242b1a..30a3262207 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; import com.hivemq.client.mqtt.MqttClient; -import com.hivemq.client.mqtt.datatypes.MqttQos; import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient; import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.io.File; @@ -103,19 +102,19 @@ public void start() { mqttDefaultSettings.put( "commonCollection", "false" ); mqttDefaultSettings.put( "commonCollectionName", "" ); mqttDefaultSettings.put( "Query Interface Name", "mqtt" ); - QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamServer.class, mqttDefaultSettings ); + QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamClient.class, mqttDefaultSettings ); } @Override public void stop() { - QueryInterfaceManager.removeInterfaceType( MqttStreamServer.class ); + QueryInterfaceManager.removeInterfaceType( MqttStreamClient.class ); } @Slf4j @Extension - public static class MqttStreamServer extends QueryInterface { + public static class MqttStreamClient extends QueryInterface { @SuppressWarnings("WeakerAccess") public static final String INTERFACE_NAME = "MQTT Interface"; @@ -167,7 +166,7 @@ public static class MqttStreamServer extends QueryInterface { private final MonitoringPage monitoringPage; - public MqttStreamServer( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { + public MqttStreamClient( TransactionManager transactionManager, Authenticator authenticator, int ifaceId, String uniqueName, Map settings ) { super( transactionManager, authenticator, ifaceId, uniqueName, settings, true, false ); // Add information page this.monitoringPage = new MonitoringPage(); @@ -228,6 +227,7 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) + .automaticReconnectWithDefaultConfig() .sslConfig() //TODO: delete or enter password from GUI password thinghere and in method .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ) @@ -239,6 +239,7 @@ public void run() { .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) + .automaticReconnectWithDefaultConfig() .buildAsync(); } @@ -571,8 +572,8 @@ protected void processMsg( Mqtt5Publish subMsg ) { String topic = subMsg.getTopic().toString(); String message = extractPayload( subMsg ); - MqttMessage mqttMessage = new MqttMessage( message, topic ); addMessageToQueue( topic, message ); + MqttMessage mqttMessage = new MqttMessage( message, topic ); String wildcardTopic = ""; if ( !topicsMap.containsKey( topic ) ) { @@ -584,36 +585,38 @@ protected void processMsg( Mqtt5Publish subMsg ) { if ( this.filterMap.containsKey( topic ) ) { String filterQuery = this.filterMap.get( topic ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, statement ); // false is returned when a message should not be stored in DB if ( streamProcessor.applyFilter() ) { - insert( mqttMessage, transaction ); + insertInEntity( mqttMessage, transaction ); } } else if ( !wildcardTopic.isEmpty() && this.filterMap.containsKey( wildcardTopic ) ) { String filterQuery = this.filterMap.get( wildcardTopic ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, statement ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, statement ); if ( streamProcessor.applyFilter() ) { - insert( mqttMessage, transaction ); + insertInEntity( mqttMessage, transaction ); } } else { - insert( mqttMessage, transaction ); + insertInEntity( mqttMessage, transaction ); } } - private void insert( MqttMessage mqttMessage, Transaction transaction ) { - ReceivedMqttMessage receivedMqttMessage; + private void insertInEntity( MqttMessage mqttMessage, Transaction transaction ) { + StoringMqttMessage storingMqttMessage; synchronized ( settingsLock ) { if ( !this.commonCollection.get() ) { String collectionToBeSaved; collectionToBeSaved = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); + storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); } else { - receivedMqttMessage = new ReceivedMqttMessage( mqttMessage, this.namespaceName, getNamespaceId( this.namespaceName, this.namespaceType ), this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.commonCollectionName ); + storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.commonCollectionName ); } } StreamCapture streamCapture = new StreamCapture( transaction ); - streamCapture.insert( receivedMqttMessage ); + streamCapture.insert( storingMqttMessage ); } @@ -694,9 +697,13 @@ private void createStreamCollection( String collectionName ) { } try { List dataStores = new ArrayList<>(); - //TODO: StreamCollection einbinden - DdlManager.getInstance().createCollection( namespaceID, collectionName, true, //only creates collection if it does not already exist. - dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); + DdlManager.getInstance().createCollection( + namespaceID, + collectionName, + true, //only creates collection if it does not already exist. + dataStores.size() == 0 ? null : dataStores, + PlacementType.MANUAL, + statement ); transaction.commit(); } catch ( EntityAlreadyExistsException | TransactionException e ) { throw new RuntimeException( "Error while creating a new collection:", e ); @@ -935,19 +942,18 @@ public MonitoringPage() { informationGroupPub = new InformationGroup( informationPage, "Publish a message" ).setOrder( 3 ); im.addGroup( informationGroupPub ); - msgButton = new InformationAction( informationGroupPub, "Send a msg", ( parameters ) -> { - String end = "Msg was published!"; + msgButton = new InformationAction( informationGroupPub, "Publish", ( parameters ) -> { + String end = "Message was published!"; try { client.publishWith() .topic( parameters.get( "topic" ) ) - .payload( parameters.get( "msg" ).getBytes() ) - .qos( MqttQos.AT_LEAST_ONCE ) + .payload( parameters.get( "payload" ).getBytes() ) .send(); } catch ( IllegalArgumentException e ) { throw new RuntimeException( e ); } return end; - } ).withParameters( "topic", "msg" ); + } ).withParameters( "topic", "payload" ); im.registerInformation( msgButton ); // Reconnection button diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index 81b0258755..7f2108db94 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -43,14 +43,13 @@ public class MqttStreamProcessor extends StreamProcessorImpl { private final Statement statement; - public MqttStreamProcessor( MqttMessage mqttMessage, String filterQuery, Statement statement ) { - super( mqttMessage.getMessage() ); - this.filterQuery = filterQuery; + public MqttStreamProcessor( FilteringMqttMessage filteringMqttMessage, Statement statement ) { + super( filteringMqttMessage.getMessage() ); + this.filterQuery = filteringMqttMessage.getQuery(); this.statement = statement; } - //TODO: in Tutorial schreiben, was allgemein für Strings gilt public boolean applyFilter() { AlgRoot root = processMqlQuery(); List> res = executeAndTransformPolyAlg( root, statement ); @@ -79,9 +78,9 @@ private AlgRoot processMqlQuery() { } else if ( isNumber( msg ) ) { double value = Double.parseDouble( msg ); msgDoc = new BsonDocument( "$$ROOT", new BsonDouble( value ) ); - } else if ( isBoolean( msg ) ) { + /*} else if ( isBoolean( msg ) ) { boolean value = Boolean.parseBoolean( msg ); - msgDoc = new BsonDocument( "$$ROOT", new BsonBoolean( value ) ); + msgDoc = new BsonDocument( "$$ROOT", new BsonBoolean( value ) );*/ } else { // msg is String msgDoc = new BsonDocument( "$$ROOT", new BsonString( msg ) ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java similarity index 71% rename from plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java rename to plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java index 5cbd46ebe9..9feddde460 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/ReceivedMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java @@ -16,19 +16,15 @@ package org.polypheny.db.mqtt; -import javax.management.Query; import lombok.Getter; -import lombok.Setter; import org.polypheny.db.catalog.Catalog.NamespaceType; //TODO: javadoc -public class ReceivedMqttMessage { +public class StoringMqttMessage { private final MqttMessage msg; @Getter private final String namespaceName; @Getter - private final long namespaceId; - @Getter private final NamespaceType namespaceType; @Getter private final String uniqueNameOfInterface; @@ -37,28 +33,21 @@ public class ReceivedMqttMessage { @Getter private final int userId; /** - * if MqttStreamServer.collectionPerTopic = TRUE, then collectionName is name of the topic or (if the subscribed topic + * if MqttStreamClient.collectionPerTopic = TRUE, then collectionName is name of the topic or (if the subscribed topic * has wildcards) the wildcardTopic - * if MqttStreamServer.collectionPerTopic = FALSE, then collectionName is the name of the common collection + * if MqttStreamClient.collectionPerTopic = FALSE, then collectionName is the name of the common collection */ @Getter - private final String collectionName; + private final String entityName; - public ReceivedMqttMessage( MqttMessage msg, String namespaceName, long namespaceId, NamespaceType namespaceType, String uniqueNameOfInterface, long databaseId, int userId, String collectionName ) { + public StoringMqttMessage( MqttMessage msg, String namespaceName, NamespaceType namespaceType, String uniqueNameOfInterface, long databaseId, int userId, String entityName ) { this.msg = msg; this.namespaceName = namespaceName; this.namespaceType = namespaceType; - //TODO: schauen, wo namespaceId gebraucht wird. - this.namespaceId = namespaceId; this.uniqueNameOfInterface = uniqueNameOfInterface; this.databaseId = databaseId; this.userId = userId; - this.collectionName = collectionName; - } - - - public String getTopic() { - return this.msg.getTopic(); + this.entityName = entityName; } @@ -66,4 +55,8 @@ public String getMessage() { return this.msg.getMessage(); } + public String getTopic() { + return this.msg.getTopic(); + } + } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index ece69b616c..a474cfc409 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -18,8 +18,13 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.bson.BsonArray; +import org.bson.BsonBoolean; import org.bson.BsonDocument; +import org.bson.BsonDouble; +import org.bson.BsonInt32; import org.bson.BsonString; +import org.bson.BsonValue; import org.polypheny.db.PolyImplementation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; @@ -36,7 +41,7 @@ public class StreamCapture { Transaction transaction; PolyphenyHomeDirManager homeDirManager; - ReceivedMqttMessage receivedMqttMessage; + StoringMqttMessage storingMqttMessage; StreamCapture( final Transaction transaction ) { @@ -44,23 +49,40 @@ public class StreamCapture { } - public void insert( ReceivedMqttMessage receivedMqttMessage ) { - this.receivedMqttMessage = receivedMqttMessage; + public void insert( StoringMqttMessage storingMqttMessage ) { + this.storingMqttMessage = storingMqttMessage; insertMessage(); } private void insertMessage() { - String sqlCollectionName = this.receivedMqttMessage.getNamespaceName() + "." + this.receivedMqttMessage.getCollectionName(); + String sqlCollectionName = this.storingMqttMessage.getNamespaceName() + "." + this.storingMqttMessage.getEntityName(); Statement statement = transaction.createStatement(); // Builder which allows to construct the algebra tree which is equivalent to query and is executed AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); BsonDocument document = new BsonDocument(); - document.put( "source", new BsonString( this.receivedMqttMessage.getUniqueNameOfInterface() ) ); - document.put( "topic", new BsonString( this.receivedMqttMessage.getTopic() ) ); - document.put( "content", new BsonString( this.receivedMqttMessage.getMessage() ) ); + document.put( "source", new BsonString( this.storingMqttMessage.getUniqueNameOfInterface() ) ); + document.put( "topic", new BsonString( this.storingMqttMessage.getTopic() ) ); + String msg = this.storingMqttMessage.getMessage(); + BsonValue value; + if ( msg.contains( "{" ) && msg.contains( "}" ) ) { + value = BsonDocument.parse( msg ); + } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { + BsonArray bsonArray = new BsonArray(); + msg = msg.replace( "[", "" ).replace( "]", "" ); + String[] msglist = msg.split( "," ); + for ( String stringValue : msglist ) { + stringValue = stringValue.trim(); + bsonArray.add( getBsonValue( stringValue ) ); + } + value = bsonArray; + } else { + // msg is a single value + value = getBsonValue( msg ); + } + document.put( "payload", value ); AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); @@ -75,6 +97,51 @@ private void insertMessage() { } + /** + * turns one single value into the corresponding BsonValue + * @param value value that has to be casted as String + * @return + */ + protected BsonValue getBsonValue(String value) { + if ( isInteger( value ) ) { + return new BsonInt32(Integer.parseInt( value ) ); + } else if ( isDouble( value ) ) { + return new BsonDouble(Double.parseDouble( value ) ); + } else if ( isBoolean( value ) ) { + return new BsonBoolean( Boolean.parseBoolean( value ) ); + } else { + return new BsonString( value ); + } + } + + + public boolean isDouble( String value ) { + try { + Double.parseDouble( value ); + } catch ( NumberFormatException e ) { + return false; + } + return true; + } + + + protected boolean isInteger( String value ) { + try { + int intNumber = Integer.parseInt( value ); + double doubleNumber = Double.parseDouble( value ); + return intNumber == doubleNumber; + } catch ( NumberFormatException e ) { + return false; + } + } + + + public boolean isBoolean( String value ) { + return value.equals( "true" ) || value.equals( "false" ); + } + + + List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { try { diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index 618fec7a3c..b363d3ca93 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -37,7 +37,7 @@ import org.polypheny.db.TestHelper; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; -import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamServer; +import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamClient; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; @@ -48,7 +48,7 @@ public class MqttStreamClientTest { static Map initialSettings = new HashMap<>(); static Map changedSettings = new HashMap<>(); - MqttStreamServer client; + MqttStreamClient client; @BeforeClass @@ -75,7 +75,7 @@ public void resetSettings() { QueryInterface iface = QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); - client = new MqttStreamServer( + client = new MqttStreamClient( transactionManager, null, iface.getQueryInterfaceId(), @@ -92,11 +92,6 @@ public void resetSettings() { } - @Test - public void UITest() { // TODO: UI komponenten testen - } - - @Test public void saveQueryEmptyStringTest() { changedSettings.replace( "filterQuery", " " ); @@ -275,6 +270,11 @@ public void processMsgTest() { MqttUserPropertiesImplBuilder.Default defaultProperties = new Default(); Mqtt5Publish message = new MqttPublish( MqttTopicImpl.of( "topic1" ), ByteBuffer.wrap( "payload".getBytes() ), MqttQos.AT_LEAST_ONCE, false, 10, null, null, null, null, defaultProperties.build(), null ); client.processMsg( message ); + //TODO: was prüfe ich hier???? + // ob die zwei Maps richtig gestzt wurden. + String[] messageInQueue = client.getMessageQueue().peek(); + assertEquals( "topic1", messageInQueue[0] ); + assertEquals( "payload", messageInQueue[1] ); } } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java index ae9609b4ed..10aee65c05 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java @@ -42,9 +42,9 @@ public void filterTestForSingleNumberMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; - MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue(streamProcessor.applyFilter()); } @@ -54,10 +54,10 @@ public void filterTestForSingleNumberMessage2() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; - - MqttMessage mqttMessage1 = new MqttMessage( "15", "button/battery" ); - MqttStreamProcessor streamProcessor1 = new MqttStreamProcessor( mqttMessage1, filterQuery, st ); - assertFalse(streamProcessor1.applyFilter()); + MqttMessage mqttMessage = new MqttMessage( "15", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); + assertFalse(streamProcessor.applyFilter()); } @@ -67,10 +67,10 @@ public void filterTestForSingleStringMessage1() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":\"shouldMatch\"}"; - - MqttMessage mqttMessage2 = new MqttMessage( "shouldMatch", "button/battery" ); - MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( mqttMessage2, filterQuery, st ); - assertTrue(streamProcessor2.applyFilter()); + MqttMessage mqttMessage = new MqttMessage( "shouldMatch", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); + assertTrue(streamProcessor.applyFilter()); } @@ -80,8 +80,9 @@ public void filterTestForSingleStringMessage2() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":\"shouldNot\"}"; - MqttMessage mqttMessage2 = new MqttMessage( "shouldNotMatch", "button/battery" ); - MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( mqttMessage2, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage( "shouldNotMatch", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( filteringMqttMessage, st ); assertFalse(streamProcessor2.applyFilter()); } @@ -92,7 +93,8 @@ public void filterTestForArrayMessage() { Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; MqttMessage mqttMessage = new MqttMessage( "[10]", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter() ); } @@ -103,7 +105,8 @@ public void filterTestForArrayMessage2() { Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":[10]}"; MqttMessage mqttMessage = new MqttMessage( "[10]", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter() ); } @@ -113,8 +116,9 @@ public void filterTestForArrayMessageFalse() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; - MqttMessage mqttMessage = new MqttMessage( "[15, 14]", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage( "[15, 14]", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertFalse( streamProcessor.applyFilter()); } @@ -123,17 +127,19 @@ public void filterTestForBooleanMessageTrue() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":true}"; - MqttMessage mqttMessage = new MqttMessage( "true", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); - assertFalse( streamProcessor.applyFilter()); + MqttMessage mqttMessage = new MqttMessage("true", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); + assertTrue( streamProcessor.applyFilter()); } @Test public void filterTestForBooleanMessageFalse() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":true}"; - MqttMessage mqttMessage = new MqttMessage( "false", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage("false", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertFalse( streamProcessor.applyFilter()); } @@ -143,8 +149,9 @@ public void filterTestForJsonNumberMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"count\":10}"; - MqttMessage mqttMessage = new MqttMessage( "{\"count\":10}", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage( "{\"count\":10}", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter()); } @@ -154,8 +161,9 @@ public void filterTestForJsonArrayMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"array\":10}"; - MqttMessage mqttMessage = new MqttMessage( "{\"array\":[10]}", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage( "{\"array\":[10]}", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter()); } @@ -165,8 +173,9 @@ public void filterTestForJsonStringMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"content\":\"online\"}"; - MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage("{\"content\":\"online\"}", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter()); } @@ -175,8 +184,9 @@ public void filterTestForJsonStringMessage2() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"content\":\"online\"}"; - MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); - MqttStreamProcessor streamProcessor = new MqttStreamProcessor( mqttMessage, filterQuery, st ); + MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); assertTrue( streamProcessor.applyFilter()); } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java new file mode 100644 index 0000000000..3977b2ca0c --- /dev/null +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import static org.junit.Assert.assertTrue; + +import com.hivemq.client.internal.mqtt.datatypes.MqttTopicImpl; +import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder; +import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder.Default; +import com.hivemq.client.internal.mqtt.message.publish.MqttPublish; +import com.hivemq.client.mqtt.datatypes.MqttQos; +import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; +import java.nio.ByteBuffer; +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.transaction.Transaction; + +public class StreamCaptureTest { + + + static Transaction transaction; + static StreamCapture capture; + @BeforeClass + public static void init() { + TestHelper testHelper = TestHelper.getInstance(); + //transactionManager = testHelper.getTransactionManager(); + transaction = testHelper.getTransaction(); + capture = new StreamCapture( transaction ); + } + + + + @Test + public void numberTest() { + MqttMessage msg = new MqttMessage( "25", "battery" ); + //StoringMqttMessage receivedmsg = new StoringMqttMessage( msg, ) + //TODO: capture.insert( ); + } + + @Test + public void intTest() { + assertTrue(capture.isInteger( "1" )); + } + +} From d763c089121a0e880240e32d9b7fa5946618f391 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 25 Sep 2023 16:27:19 +0200 Subject: [PATCH 092/114] fixed find with boolean --- .../polypheny/db/runtime/functions/MqlFunctions.java | 2 ++ .../org/polypheny/db/mqtt/MqttStreamProcessor.java | 4 ++-- .../polypheny/db/mqtt/MqttStreamProcessorTest.java | 12 ++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/runtime/functions/MqlFunctions.java b/core/src/main/java/org/polypheny/db/runtime/functions/MqlFunctions.java index ab7468ff15..d3e3cc5672 100644 --- a/core/src/main/java/org/polypheny/db/runtime/functions/MqlFunctions.java +++ b/core/src/main/java/org/polypheny/db/runtime/functions/MqlFunctions.java @@ -781,6 +781,8 @@ private static Object transformBsonToPrimitive( BsonValue doc ) { return doc.asDocument().entrySet().stream().collect( Collectors.toMap( Entry::getKey, e -> transformBsonToPrimitive( e.getValue() ) ) ); case ARRAY: return doc.asArray().stream().map( MqlFunctions::transformBsonToPrimitive ).collect( Collectors.toList() ); + case BOOLEAN: + return doc.asBoolean().getValue(); default: return null; } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index 7f2108db94..ba12ca1d96 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -78,9 +78,9 @@ private AlgRoot processMqlQuery() { } else if ( isNumber( msg ) ) { double value = Double.parseDouble( msg ); msgDoc = new BsonDocument( "$$ROOT", new BsonDouble( value ) ); - /*} else if ( isBoolean( msg ) ) { + } else if ( isBoolean( msg ) ) { boolean value = Boolean.parseBoolean( msg ); - msgDoc = new BsonDocument( "$$ROOT", new BsonBoolean( value ) );*/ + msgDoc = new BsonDocument( "$$ROOT", new BsonBoolean( value ) ); } else { // msg is String msgDoc = new BsonDocument( "$$ROOT", new BsonString( msg ) ); diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java index 10aee65c05..fcb2a871de 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java @@ -190,5 +190,17 @@ public void filterTestForJsonStringMessage2() { assertTrue( streamProcessor.applyFilter()); } +//TODO: remove this test: + @Test + public void nestedDoctest() { + Transaction transaction = TestHelper.getInstance().getTransaction(); + Statement st = transaction.createStatement(); + String filterQuery = "{\"mqtt.status\":\"online\"}"; + MqttMessage mqttMessage = new MqttMessage( "{\"mqtt\":{\"status\":\"online\"}}", "button/battery"); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); + assertTrue( streamProcessor.applyFilter()); + } + } From d24a2427fca68d304224231844f7ce8289209b14 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 25 Sep 2023 21:14:59 +0200 Subject: [PATCH 093/114] defined Helper class for StreamCapture testing --- .../polypheny/db/mqtt/StreamCaptureTest.java | 134 +++++++++++++++++- 1 file changed, 129 insertions(+), 5 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java index 3977b2ca0c..6b9515a3c0 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java @@ -16,7 +16,9 @@ package org.polypheny.db.mqtt; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.reflections.Reflections.log; import com.hivemq.client.internal.mqtt.datatypes.MqttTopicImpl; import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder; @@ -25,31 +27,72 @@ import com.hivemq.client.mqtt.datatypes.MqttQos; import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.bson.BsonDocument; import org.junit.BeforeClass; import org.junit.Test; import org.polypheny.db.TestHelper; +import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.algebra.AlgNode; +import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.PlacementType; +import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; +import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; +import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; +import org.polypheny.db.ddl.DdlManager; +import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.languages.QueryParameters; +import org.polypheny.db.languages.mql.MqlFind; +import org.polypheny.db.languages.mql.MqlQueryParameters; +import org.polypheny.db.languages.mql2alg.MqlToAlgConverter; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.prepare.PolyphenyDbCatalogReader; +import org.polypheny.db.processing.Processor; +import org.polypheny.db.tools.AlgBuilder; +import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; +import org.polypheny.db.transaction.TransactionManager; public class StreamCaptureTest { - + static TransactionManager transactionManager; static Transaction transaction; + static Statement statement; static StreamCapture capture; + + static long namespaceId; @BeforeClass public static void init() { TestHelper testHelper = TestHelper.getInstance(); - //transactionManager = testHelper.getTransactionManager(); + transactionManager = testHelper.getTransactionManager(); transaction = testHelper.getTransaction(); + statement = transaction.createStatement(); capture = new StreamCapture( transaction ); + namespaceId = Helper.createNamespace("testspace", NamespaceType.DOCUMENT); + Helper.createCollection(); } + + + + + @Test public void numberTest() { - MqttMessage msg = new MqttMessage( "25", "battery" ); - //StoringMqttMessage receivedmsg = new StoringMqttMessage( msg, ) - //TODO: capture.insert( ); + MqttMessage msg = new MqttMessage( "25", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + capture.insert( storingmsg ); + BsonDocument doc = Helper.filter( "{\"payload\":25}" ).get( 0 ); + assertEquals( "testTopic", doc.get( "topic" ).asString().getValue() ); + assertEquals( 25, doc.get( "payload" ).asNumber().intValue() ); + assertEquals( "streamCaptureTest", doc.get( "source" ).asString().getValue() ); } @Test @@ -57,4 +100,85 @@ public void intTest() { assertTrue(capture.isInteger( "1" )); } + + private static class Helper { + private static long createNamespace( String namespaceName, NamespaceType namespaceType ) { + Catalog catalog = Catalog.getInstance(); + long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); + try { + catalog.commit(); + return id; + } catch ( NoTablePrimaryKeyException e ) { + throw new RuntimeException( e ); + } + } + + + private static void createCollection() { + Statement statement = transaction.createStatement(); + try { + List dataStores = new ArrayList<>(); + DdlManager.getInstance().createCollection( + namespaceId, + "testCollection", + true, + dataStores.size() == 0 ? null : dataStores, + PlacementType.MANUAL, + statement ); + transaction.commit(); + } catch ( Exception | TransactionException e ) { + throw new RuntimeException( "Error while creating a new collection:", e ); + } + } + + + private static List scanCollection() { + String sqlCollectionName = "testSpace" + "." + "testCollection"; + Statement statement = transaction.createStatement(); + AlgBuilder builder = AlgBuilder.create( statement ); + + AlgNode algNode = builder.docScan( statement, sqlCollectionName ).build(); + + AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); + List> res = capture.executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + List result = new ArrayList<>(); + for ( List objectsList : res ) { + result.add( objectsList.get( 0 ).toString() ); + } + return result; + } + + + private static List filter(String query) { + QueryParameters parameters = new MqlQueryParameters( query, "testSpace", NamespaceType.DOCUMENT ); + AlgBuilder algBuilder = AlgBuilder.create( statement ); + Processor mqlProcessor = transaction.getProcessor( QueryLanguage.from( "mongo" ) ); + PolyphenyDbCatalogReader catalogReader = transaction.getCatalogReader(); + final AlgOptCluster cluster = AlgOptCluster.createDocument( statement.getQueryProcessor().getPlanner(), algBuilder.getRexBuilder() ); + MqlToAlgConverter mqlConverter = new MqlToAlgConverter( mqlProcessor, catalogReader, cluster ); + + MqlFind find = (MqlFind) mqlProcessor.parse( String.format( "db.%s.find(%s)", "testCollection", query ) ).get( 0 ); + + AlgRoot root = mqlConverter.convert( find, parameters ); + + List> res = capture.executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + List result = new ArrayList<>(); + for ( List objectsList : res ) { + result.add( objectsList.get( 0 ).toString() ); + } + + List listOfMessage = new ArrayList<>(); + for ( String documentString : result ) { + BsonDocument doc = BsonDocument.parse(documentString); + listOfMessage.add( doc ); + } + return listOfMessage; + } + + + + } + + + } From 5610f8f945829d80cff95244aa9a374d0b08fef3 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 25 Sep 2023 21:15:36 +0200 Subject: [PATCH 094/114] changed variable name message -> payload --- plugins/mqtt-stream/build.gradle | 1 + .../java/org/polypheny/db/mqtt/MqttMessage.java | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 46e5db7d92..1946cd6cae 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -14,6 +14,7 @@ dependencies { // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: hivemq_mqttclient_version + //TODO: remove // https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.70' diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java index 7462e86bb0..9d06a5ddab 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java @@ -21,15 +21,20 @@ public class MqttMessage implements StreamMessage { - @Getter - final String message; + + final String payload; @Getter final String topic; - public MqttMessage( String message, String topic ) { - this.message = message; + public MqttMessage( String payload, String topic ) { + this.payload = payload; this.topic = topic; } + + public String getMessage() { + return this.payload; + } + } From c359ddfce3777c7f73a61c1315a393ad61539c3b Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Wed, 27 Sep 2023 14:03:41 +0200 Subject: [PATCH 095/114] Debugged code while testing stream capture + mqttClient component --- .../db/mqtt/FilteringMqttMessage.java | 3 + .../polypheny/db/mqtt/MqttStreamPlugin.java | 6 +- .../polypheny/db/mqtt/StoringMqttMessage.java | 3 + .../org/polypheny/db/mqtt/StreamCapture.java | 11 +- .../db/mqtt/MqttStreamClientTest.java | 73 +++++---- .../db/mqtt/MqttStreamProcessorTest.java | 71 +++++---- .../polypheny/db/mqtt/StreamCaptureTest.java | 147 ++++++++++++++---- 7 files changed, 207 insertions(+), 107 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java index f8c9b0c8ad..900cc56de7 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java @@ -19,9 +19,12 @@ import lombok.Getter; public class FilteringMqttMessage { + private MqttMessage mqttMessage; @Getter private String query; + + public FilteringMqttMessage( MqttMessage mqttMessage, String query ) { this.mqttMessage = mqttMessage; this.query = query; diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 30a3262207..745e3506b5 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -620,12 +620,12 @@ private void insertInEntity( MqttMessage mqttMessage, Transaction transaction ) } - private static String extractPayload( Mqtt5Publish subMsg ) { + protected static String extractPayload( Mqtt5Publish subMsg ) { return new String( subMsg.getPayloadAsBytes(), Charset.defaultCharset() ); } - private String getWildcardTopic( String topic ) { + protected String getWildcardTopic( String topic ) { for ( String t : topicsMap.keySet() ) { //multilevel wildcard if ( t.contains( "#" ) && topic.startsWith( t.substring( 0, t.indexOf( "#" ) ) ) ) { @@ -641,7 +641,7 @@ private String getWildcardTopic( String topic ) { } - private void addMessageToQueue( String topic, String message ) { + protected void addMessageToQueue( String topic, String message ) { if ( this.messageQueue.size() >= 20 ) { this.messageQueue.poll(); this.messageQueue.add( new String[]{ topic, message } ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java index 9feddde460..ff1fbc8398 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java @@ -18,6 +18,7 @@ import lombok.Getter; import org.polypheny.db.catalog.Catalog.NamespaceType; + //TODO: javadoc public class StoringMqttMessage { @@ -40,6 +41,7 @@ public class StoringMqttMessage { @Getter private final String entityName; + public StoringMqttMessage( MqttMessage msg, String namespaceName, NamespaceType namespaceType, String uniqueNameOfInterface, long databaseId, int userId, String entityName ) { this.msg = msg; this.namespaceName = namespaceName; @@ -55,6 +57,7 @@ public String getMessage() { return this.msg.getMessage(); } + public String getTopic() { return this.msg.getTopic(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index a474cfc409..3c4774d1e4 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -99,14 +99,15 @@ private void insertMessage() { /** * turns one single value into the corresponding BsonValue + * * @param value value that has to be casted as String * @return */ - protected BsonValue getBsonValue(String value) { + protected BsonValue getBsonValue( String value ) { if ( isInteger( value ) ) { - return new BsonInt32(Integer.parseInt( value ) ); + return new BsonInt32( Integer.parseInt( value ) ); } else if ( isDouble( value ) ) { - return new BsonDouble(Double.parseDouble( value ) ); + return new BsonDouble( Double.parseDouble( value ) ); } else if ( isBoolean( value ) ) { return new BsonBoolean( Boolean.parseBoolean( value ) ); } else { @@ -125,7 +126,7 @@ public boolean isDouble( String value ) { } - protected boolean isInteger( String value ) { + public boolean isInteger( String value ) { try { int intNumber = Integer.parseInt( value ); double doubleNumber = Double.parseDouble( value ); @@ -141,11 +142,9 @@ public boolean isBoolean( String value ) { } - List> executeAndTransformPolyAlg( AlgRoot algRoot, Statement statement, final Context ctx ) { try { - // Prepare PolyImplementation result = statement.getQueryProcessor().prepareQuery( algRoot, false ); log.debug( "AlgRoot was prepared." ); diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index b363d3ca93..3fc8361326 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -20,17 +20,11 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.hivemq.client.internal.mqtt.datatypes.MqttTopicImpl; -import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder; -import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder.Default; -import com.hivemq.client.internal.mqtt.message.publish.MqttPublish; -import com.hivemq.client.mqtt.datatypes.MqttQos; -import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -103,42 +97,52 @@ public void saveQueryEmptyStringTest() { @Test public void saveSimpleQueryTest() { - changedSettings.replace( "filterQuery", "topic1:{key1:value1}" ); + changedSettings.replace( "filterQuery", "topic1:{\"key1\":\"value1\"}" ); client.updateSettings( changedSettings ); Map expected = new HashMap<>( 1 ); - expected.put( "topic1", "{key1:value1}" ); + expected.put( "topic1", "{\"key1\":\"value1\"}" ); + assertEquals( expected, client.getFilterMap() ); + } + + + @Test + public void saveQueryToExistingTopicTest() { + changedSettings.replace( "filterQuery", "topic1:{\"key1\":\"value2\"}" ); + client.updateSettings( changedSettings ); + Map expected = new HashMap<>( 1 ); + expected.put( "topic1", "{\"key1\":\"value2\"}" ); assertEquals( expected, client.getFilterMap() ); } @Test public void saveQueryWithArrayTest() { - changedSettings.replace( "filterQuery", "topic1:{key1:[1, 2, 3]}" ); + changedSettings.replace( "filterQuery", "topic1:{\"key1\":[1, 2, 3]}" ); client.updateSettings( changedSettings ); Map expected = new HashMap<>( 1 ); - expected.put( "topic1", "{key1:[1, 2, 3]}" ); + expected.put( "topic1", "{\"key1\":[1, 2, 3]}" ); assertEquals( expected, client.getFilterMap() ); } @Test public void saveTwoSimpleQueryTest() { - changedSettings.replace( "filterQuery", "topic1:{key1:value1}, topic2:{key2:value2}" ); + changedSettings.replace( "filterQuery", "topic1:{\"key1\":\"value1\"}, topic2:{\"key2\":\"value2\"}" ); client.updateSettings( changedSettings ); Map expected = new HashMap<>( 2 ); - expected.put( "topic1", "{key1:value1}" ); - expected.put( "topic2", "{key2:value2}" ); + expected.put( "topic1", "{\"key1\":\"value1\"}" ); + expected.put( "topic2", "{\"key2\":\"value2\"}" ); assertEquals( expected, client.getFilterMap() ); } @Test public void saveNestedQueryTest() { - changedSettings.replace( "filterQuery", "topic1:{key1:{$lt:3}}, topic2:{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); + changedSettings.replace( "filterQuery", "topic1:{\"key1\":{$lt:3}}, topic2:{$or:[\"key2\":{$lt:3}, \"key2\":{$gt:5}]}" ); client.updateSettings( changedSettings ); Map expected = new HashMap<>( 2 ); - expected.put( "topic1", "{key1:{$lt:3}}" ); - expected.put( "topic2", "{$or:[key2:{$lt:3}, key2:{$gt:5}]}" ); + expected.put( "topic1", "{\"key1\":{$lt:3}}" ); + expected.put( "topic2", "{$or:[\"key2\":{$lt:3}, \"key2\":{$gt:5}]}" ); assertEquals( expected, client.getFilterMap() ); } @@ -170,9 +174,26 @@ public void toListWithContentTest() { @Test - public void reloadSettingsTopicTest() { - //TODO with broker - //TODO: with wildcards + public void addFirstMessageToQueueTest() { + ConcurrentLinkedQueue msgQueueBefore = client.getMessageQueue(); + assertEquals( 0, msgQueueBefore.size() ); + client.addMessageToQueue( "topic1", "payload1" ); + ConcurrentLinkedQueue msgQueueAfter = client.getMessageQueue(); + assertEquals( 1, msgQueueAfter.size() ); + String[] expected = { "topic1", "payload1" }; + assertEquals( expected, msgQueueAfter.poll() ); + } + + + @Test + public void addTwentyOneMessagesToQueueTest() { + for ( int i = 0; i < 22; i++ ) { + client.addMessageToQueue( "topic1", String.valueOf( i ) ); + } + ConcurrentLinkedQueue msgQueueAfter = client.getMessageQueue(); + assertEquals( 20, msgQueueAfter.size() ); + String[] expected = { "topic1", String.valueOf( 2 ) }; + assertEquals( expected, msgQueueAfter.poll() ); } @@ -265,16 +286,4 @@ public void reloadSettingsCommonCollectionAndCommonCollectionNameTest2() { } - @Test - public void processMsgTest() { - MqttUserPropertiesImplBuilder.Default defaultProperties = new Default(); - Mqtt5Publish message = new MqttPublish( MqttTopicImpl.of( "topic1" ), ByteBuffer.wrap( "payload".getBytes() ), MqttQos.AT_LEAST_ONCE, false, 10, null, null, null, null, defaultProperties.build(), null ); - client.processMsg( message ); - //TODO: was prüfe ich hier???? - // ob die zwei Maps richtig gestzt wurden. - String[] messageInQueue = client.getMessageQueue().peek(); - assertEquals( "topic1", messageInQueue[0] ); - assertEquals( "payload", messageInQueue[1] ); - } - } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java index fcb2a871de..5de06de51d 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamProcessorTest.java @@ -16,13 +16,9 @@ package org.polypheny.db.mqtt; -import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; -import java.util.ArrayList; -import java.util.List; -import org.bson.BsonDouble; -import org.bson.BsonValue; import org.junit.BeforeClass; import org.junit.Test; import org.polypheny.db.TestHelper; @@ -37,6 +33,7 @@ public static void init() { } + @Test public void filterTestForSingleNumberMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); @@ -45,19 +42,20 @@ public void filterTestForSingleNumberMessage() { MqttMessage mqttMessage = new MqttMessage( "10", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue(streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } + @Test public void filterTestForSingleNumberMessage2() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; - MqttMessage mqttMessage = new MqttMessage( "15", "button/battery" ); + MqttMessage mqttMessage = new MqttMessage( "15", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertFalse(streamProcessor.applyFilter()); + assertFalse( streamProcessor.applyFilter() ); } @@ -67,10 +65,10 @@ public void filterTestForSingleStringMessage1() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":\"shouldMatch\"}"; - MqttMessage mqttMessage = new MqttMessage( "shouldMatch", "button/battery" ); + MqttMessage mqttMessage = new MqttMessage( "shouldMatch", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue(streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } @@ -83,7 +81,7 @@ public void filterTestForSingleStringMessage2() { MqttMessage mqttMessage = new MqttMessage( "shouldNotMatch", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor2 = new MqttStreamProcessor( filteringMqttMessage, st ); - assertFalse(streamProcessor2.applyFilter()); + assertFalse( streamProcessor2.applyFilter() ); } @@ -116,31 +114,34 @@ public void filterTestForArrayMessageFalse() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":10}"; - MqttMessage mqttMessage = new MqttMessage( "[15, 14]", "button/battery"); - FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttMessage mqttMessage = new MqttMessage( "[15, 14]", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertFalse( streamProcessor.applyFilter()); + assertFalse( streamProcessor.applyFilter() ); } + @Test public void filterTestForBooleanMessageTrue() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":true}"; - MqttMessage mqttMessage = new MqttMessage("true", "button/battery"); + MqttMessage mqttMessage = new MqttMessage( "true", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } + + @Test public void filterTestForBooleanMessageFalse() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"$$ROOT\":true}"; - MqttMessage mqttMessage = new MqttMessage("false", "button/battery"); + MqttMessage mqttMessage = new MqttMessage( "false", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertFalse( streamProcessor.applyFilter()); + assertFalse( streamProcessor.applyFilter() ); } @@ -149,10 +150,10 @@ public void filterTestForJsonNumberMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"count\":10}"; - MqttMessage mqttMessage = new MqttMessage( "{\"count\":10}", "button/battery"); - FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttMessage mqttMessage = new MqttMessage( "{\"count\":10}", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } @@ -161,10 +162,10 @@ public void filterTestForJsonArrayMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"array\":10}"; - MqttMessage mqttMessage = new MqttMessage( "{\"array\":[10]}", "button/battery"); - FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttMessage mqttMessage = new MqttMessage( "{\"array\":[10]}", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } @@ -173,33 +174,35 @@ public void filterTestForJsonStringMessage() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"content\":\"online\"}"; - MqttMessage mqttMessage = new MqttMessage("{\"content\":\"online\"}", "button/battery"); + MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } + @Test public void filterTestForJsonStringMessage2() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"content\":\"online\"}"; - MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery"); - FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttMessage mqttMessage = new MqttMessage( "{\"content\":\"online\"}", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } -//TODO: remove this test: + + //TODO: remove this test: @Test public void nestedDoctest() { Transaction transaction = TestHelper.getInstance().getTransaction(); Statement st = transaction.createStatement(); String filterQuery = "{\"mqtt.status\":\"online\"}"; - MqttMessage mqttMessage = new MqttMessage( "{\"mqtt\":{\"status\":\"online\"}}", "button/battery"); - FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage(mqttMessage, filterQuery ); + MqttMessage mqttMessage = new MqttMessage( "{\"mqtt\":{\"status\":\"online\"}}", "button/battery" ); + FilteringMqttMessage filteringMqttMessage = new FilteringMqttMessage( mqttMessage, filterQuery ); MqttStreamProcessor streamProcessor = new MqttStreamProcessor( filteringMqttMessage, st ); - assertTrue( streamProcessor.applyFilter()); + assertTrue( streamProcessor.applyFilter() ); } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java index 6b9515a3c0..cf1fc8d8d2 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java @@ -18,18 +18,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.reflections.Reflections.log; - -import com.hivemq.client.internal.mqtt.datatypes.MqttTopicImpl; -import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder; -import com.hivemq.client.internal.mqtt.datatypes.MqttUserPropertiesImplBuilder.Default; -import com.hivemq.client.internal.mqtt.message.publish.MqttPublish; -import com.hivemq.client.mqtt.datatypes.MqttQos; -import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; -import java.nio.ByteBuffer; + import java.util.ArrayList; import java.util.List; +import org.bson.BsonArray; import org.bson.BsonDocument; +import org.bson.BsonInt32; import org.junit.BeforeClass; import org.junit.Test; import org.polypheny.db.TestHelper; @@ -40,9 +34,7 @@ import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; -import org.polypheny.db.catalog.exceptions.EntityAlreadyExistsException; import org.polypheny.db.catalog.exceptions.NoTablePrimaryKeyException; -import org.polypheny.db.catalog.exceptions.UnknownSchemaIdRuntimeException; import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.languages.QueryParameters; @@ -56,52 +48,142 @@ import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; -import org.polypheny.db.transaction.TransactionManager; public class StreamCaptureTest { - static TransactionManager transactionManager; - static Transaction transaction; - static Statement statement; static StreamCapture capture; - + static Transaction transaction; static long namespaceId; + + @BeforeClass public static void init() { TestHelper testHelper = TestHelper.getInstance(); - transactionManager = testHelper.getTransactionManager(); transaction = testHelper.getTransaction(); - statement = transaction.createStatement(); capture = new StreamCapture( transaction ); - namespaceId = Helper.createNamespace("testspace", NamespaceType.DOCUMENT); + namespaceId = Helper.createNamespace( "testspace", NamespaceType.DOCUMENT ); Helper.createCollection(); } + @Test + public void insertIntTest() { + MqttMessage msg = new MqttMessage( "25", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); + capture.insert( storingmsg ); + // getting content with index 0 because there will only be one document matching to this query + BsonDocument result = Helper.filter( "{\"payload\":25}" ).get( 0 ); + System.out.println( "int Test" ); + System.out.println( result.toString() ); + + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + assertEquals( 25, result.get( "payload" ).asInt32().intValue() ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); + } + @Test + public void insertDoubleTest() { + MqttMessage msg = new MqttMessage( "25.54", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); + capture.insert( storingmsg ); + BsonDocument result = Helper.filter( "{\"payload\":25.54}" ).get( 0 ); + System.out.println( "double Test" ); + System.out.println( result.toString() ); + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + assertEquals( 25.54, result.get( "payload" ).asDouble().getValue(), 0.1 ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); + } + @Test + public void insertStringTest() { + MqttMessage msg = new MqttMessage( "String", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); + capture.insert( storingmsg ); + BsonDocument result = Helper.filter( "{\"payload\":\"String\"}" ).get( 0 ); + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + assertEquals( "String", result.get( "payload" ).asString().getValue() ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); + } @Test - public void numberTest() { - MqttMessage msg = new MqttMessage( "25", "testTopic" ); + public void insertBooleanTest() { + MqttMessage msg = new MqttMessage( "true", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); + capture.insert( storingmsg ); + BsonDocument result = Helper.filter( "{\"payload\":true}" ).get( 0 ); + System.out.println( "bool Test" ); + System.out.println( result.toString() ); + List collection = Helper.scanCollection(); + + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + assertEquals( true, result.get( "payload" ).asBoolean().getValue() ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); + } + + + @Test + public void insertJsonTest() { + MqttMessage msg = new MqttMessage( "{\"key1\":\"value1\", \"key2\":true, \"key3\":3}", "testTopic" ); + StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); + capture.insert( storingmsg ); + BsonDocument result = Helper.filter( "{\"payload.key1\":\"value1\"}" ).get( 0 ); + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); + assertEquals( "value1", result.get( "payload" ).asDocument().get( "key1" ).asString().getValue() ); + assertEquals( true, result.get( "payload" ).asDocument().get( "key2" ).asBoolean().getValue() ); + assertEquals( 3, result.get( "payload" ).asDocument().get( "key3" ).asInt32().getValue() ); + + } + + + @Test + public void insertArrayTest() { + MqttMessage msg = new MqttMessage( "[1, 2, 3]", "testTopic" ); StoringMqttMessage storingmsg = new StoringMqttMessage( msg, "testspace", NamespaceType.DOCUMENT, "streamCaptureTest", Catalog.defaultDatabaseId, Catalog.defaultUserId, "testCollection" ); + //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); - BsonDocument doc = Helper.filter( "{\"payload\":25}" ).get( 0 ); - assertEquals( "testTopic", doc.get( "topic" ).asString().getValue() ); - assertEquals( 25, doc.get( "payload" ).asNumber().intValue() ); - assertEquals( "streamCaptureTest", doc.get( "source" ).asString().getValue() ); + BsonDocument result = Helper.filter( "{\"payload\":[1, 2, 3]}" ).get( 0 ); + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); + BsonArray expectedPayload = new BsonArray(); + expectedPayload.add( 0, new BsonInt32( 1 ) ); + expectedPayload.add( 1, new BsonInt32( 2 ) ); + expectedPayload.add( 2, new BsonInt32( 3 ) ); + assertEquals( expectedPayload, result.get( "payload" ).asArray() ); + assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); } + @Test - public void intTest() { - assertTrue(capture.isInteger( "1" )); + public void isIntTest() { + //StreamCapture capture = new StreamCapture( transaction ); + assertTrue( capture.isInteger( "1" ) ); + } + + + @Test + public void isDoubleTest() { + //StreamCapture capture = new StreamCapture( transaction ); + assertTrue( capture.isDouble( "1.0" ) ); + } + + + @Test + public void isBooleanTest() { + //StreamCapture capture = new StreamCapture( transaction ); + assertTrue( capture.isBoolean( "false" ) ); } private static class Helper { + private static long createNamespace( String namespaceName, NamespaceType namespaceType ) { Catalog catalog = Catalog.getInstance(); long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); @@ -132,6 +214,7 @@ private static void createCollection() { } + //TODO: StreamCapture capture private static List scanCollection() { String sqlCollectionName = "testSpace" + "." + "testCollection"; Statement statement = transaction.createStatement(); @@ -149,8 +232,10 @@ private static List scanCollection() { } - private static List filter(String query) { - QueryParameters parameters = new MqlQueryParameters( query, "testSpace", NamespaceType.DOCUMENT ); + //TODO StreamCapture capture + private static List filter( String query ) { + Statement statement = transaction.createStatement(); + QueryParameters parameters = new MqlQueryParameters( String.format( "db.%s.find(%s)", "testCollection", query ), "testSpace", NamespaceType.DOCUMENT ); AlgBuilder algBuilder = AlgBuilder.create( statement ); Processor mqlProcessor = transaction.getProcessor( QueryLanguage.from( "mongo" ) ); PolyphenyDbCatalogReader catalogReader = transaction.getCatalogReader(); @@ -169,16 +254,14 @@ private static List filter(String query) { List listOfMessage = new ArrayList<>(); for ( String documentString : result ) { - BsonDocument doc = BsonDocument.parse(documentString); + BsonDocument doc = BsonDocument.parse( documentString ); listOfMessage.add( doc ); } return listOfMessage; } - } - } From f70e93389eaa08e0c0ec5a7567daeca2c245bb88 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 28 Sep 2023 10:03:17 +0200 Subject: [PATCH 096/114] added getter to topicsMap --- .../src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 745e3506b5..1c2edef856 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -138,6 +138,7 @@ public static class MqttStreamClient extends QueryInterface { private final String brokerAddress; @Getter private final int brokerPort; + @Getter private Map topicsMap = new ConcurrentHashMap<>(); /** * Contains the predicates that determines whether a message is inserted. From a13c18dda6ba218660305058f7a73f5b7e16885a Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 28 Sep 2023 10:09:37 +0200 Subject: [PATCH 097/114] added workflow for testing a broker in container --- .github/workflows/mqttClientTest.yml | 51 ++++++++ plugins/mqtt-stream/build.gradle | 10 ++ .../db/mqtt/MqttClientBrokerTest.java | 114 ++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 .github/workflows/mqttClientTest.yml create mode 100644 plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java diff --git a/.github/workflows/mqttClientTest.yml b/.github/workflows/mqttClientTest.yml new file mode 100644 index 0000000000..552af311e6 --- /dev/null +++ b/.github/workflows/mqttClientTest.yml @@ -0,0 +1,51 @@ +name: Polypheny-DB MQTT Client Test + +on: + push: { branches: mqtt-interface } + +jobs: + build: + runs-on: ubuntu-latest + name: MQTT Client Tests (Java 17) + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 17 + - name: Set env variable + run: | + echo "POLYPHENY_HOME=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Create folders for certs + run: | + mkdir $POLYPHENY_HOME/.polypheny + mkdir $POLYPHENY_HOME/.polypheny/certs + + - name: Start Mosquitto broker in docker container + uses: namoshek/mosquitto-github-action@v1 + with: + version: '5' + ports: '1883:1883' + container-name: 'mqttBroker' + + - name: Assemble + uses: nick-invision/retry@v2 + with: + max_attempts: 2 + timeout_minutes: 60 + command: ./gradlew assemble + + - name: Build Plugins + uses: nick-invision/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 60 + command: ./gradlew assemblePlugins + + - name: Execute MQTT client tests + uses: nick-invision/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 30 + command: ./gradlew mqttTests \ No newline at end of file diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index 1946cd6cae..a3546f4fe7 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -60,6 +60,16 @@ sourceSets { } } +task mqttTests(type: Test) { + description = 'Runs MQTT client tests.' + group = 'verification' + useJUnit { + includeCategories 'org.polypheny.db.mqtt.MqttClientBrokerTest' + } + shouldRunAfter(tasks.named('test')) +} +mqttTests.dependsOn(testClasses) + compileJava { dependsOn(":config:processResources") dependsOn(":core:processResources") diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java new file mode 100644 index 0000000000..ea6dcf9988 --- /dev/null +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mqtt; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.iface.QueryInterface; +import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamClient; +import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionManager; + +public class MqttClientBrokerTest { + + + static TransactionManager transactionManager; + static Transaction transaction; + static Map initialSettings = new HashMap<>(); + static Map changedSettings = new HashMap<>(); + static MqttStreamClient client; + + @BeforeClass + public static void init() { + TestHelper testHelper = TestHelper.getInstance(); + transactionManager = testHelper.getTransactionManager(); + transaction = testHelper.getTransaction(); + initialSettings.clear(); + initialSettings.put( "brokerAddress", "1883" ); + initialSettings.put( "brokerPort", "1883" ); + initialSettings.put( "commonCollectionName", "testCollection" ); + initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "namespace", "testNamespace" ); + initialSettings.put( "namespaceType", "DOCUMENT" ); + initialSettings.put( "topics", "button" ); + initialSettings.put( "Tsl/SslConnection", "false" ); + initialSettings.put( "filterQuery", "" ); + + QueryInterface iface = QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); + + client = new MqttStreamClient( + transactionManager, + null, + iface.getQueryInterfaceId(), + iface.getUniqueName(), + initialSettings ); + + } + +/* + @Before + public void resetSettings() { + initialSettings.clear(); + initialSettings.put( "brokerAddress", "1883" ); + initialSettings.put( "brokerPort", "1883" ); + initialSettings.put( "commonCollectionName", "testCollection" ); + initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "namespace", "testNamespace" ); + initialSettings.put( "namespaceType", "DOCUMENT" ); + initialSettings.put( "topics", "" ); + initialSettings.put( "Tsl/SslConnection", "false" ); + initialSettings.put( "filterQuery", "" ); + + QueryInterface iface = QueryInterfaceManager.getInstance().getQueryInterface( "mqtt" ); + + client = new MqttStreamClient( + transactionManager, + null, + iface.getQueryInterfaceId(), + iface.getUniqueName(), + initialSettings ); + + changedSettings.clear(); + changedSettings.put( "commonCollectionName", "testCollection" ); + changedSettings.put( "commonCollection", "true" ); + changedSettings.put( "namespace", "testNamespace" ); + changedSettings.put( "topics", "" ); + changedSettings.put( "filterQuery", "" ); + } + */ + + @Test + public void connectionTest() { + Map topicsMap = client.getTopicsMap(); + assertEquals( 1, topicsMap.size() ); + assertEquals( 0, topicsMap.get( "button" ) ); + } + + + + +} From f413f117358eb99254370135445ecf337008c6cd Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Thu, 28 Sep 2023 10:20:25 +0200 Subject: [PATCH 098/114] changed version --- .github/workflows/mqttClientTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mqttClientTest.yml b/.github/workflows/mqttClientTest.yml index 552af311e6..946ae5fd57 100644 --- a/.github/workflows/mqttClientTest.yml +++ b/.github/workflows/mqttClientTest.yml @@ -25,7 +25,7 @@ jobs: - name: Start Mosquitto broker in docker container uses: namoshek/mosquitto-github-action@v1 with: - version: '5' + version: '1.6' ports: '1883:1883' container-name: 'mqttBroker' From e4b66cc9554ba1ab0e7a4e85405bf8c7fdc73d14 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 13:51:57 +0200 Subject: [PATCH 099/114] renamed variable --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 102 +++++++++--------- .../db/mqtt/MqttStreamClientTest.java | 64 +++++------ 2 files changed, 85 insertions(+), 81 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 1c2edef856..ca317e87f4 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -99,8 +99,8 @@ public void start() { mqttDefaultSettings.put( "Tsl/SslConnection", "false" ); mqttDefaultSettings.put( "namespace", "default" ); mqttDefaultSettings.put( "namespaceType", "DOCUMENT" ); - mqttDefaultSettings.put( "commonCollection", "false" ); - mqttDefaultSettings.put( "commonCollectionName", "" ); + mqttDefaultSettings.put( "catchAllEntity", "false" ); + mqttDefaultSettings.put( "catchAllEntityName", "" ); mqttDefaultSettings.put( "Query Interface Name", "mqtt" ); QueryInterfaceManager.addInterfaceType( "mqtt", MqttStreamClient.class, mqttDefaultSettings ); } @@ -128,8 +128,8 @@ public static class MqttStreamClient extends QueryInterface { // "RELATIONAL", "GRAPH" types are not supported yet. new QueryInterfaceSettingList( "namespaceType", false, true, false, new ArrayList<>( List.of( "DOCUMENT" ) ) ), - new QueryInterfaceSettingList( "commonCollection", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), - new QueryInterfaceSettingString( "commonCollectionName", true, false, true, null ), + new QueryInterfaceSettingList( "catchAllEntity", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), + new QueryInterfaceSettingString( "catchAllEntityName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); @@ -138,13 +138,13 @@ public static class MqttStreamClient extends QueryInterface { private final String brokerAddress; @Getter private final int brokerPort; + /** + * topicsMap: Contains all subscribed topics as the key and the received number of messages with this topic. + */ @Getter private Map topicsMap = new ConcurrentHashMap<>(); /** - * Contains the predicates that determines whether a message is inserted. - * The Predicates are applied to the message in form of a filter and if this query returns true,the message will - * be inserted. If there is no predicate for a topic, then all messages are saved. - * If there are several queries for the topic of the message, they are comma seperated. + * filterMap: Contains the filter query for a topic. The key is the topic. */ @Getter private Map filterMap = new ConcurrentHashMap<>(); @@ -156,13 +156,12 @@ public static class MqttStreamClient extends QueryInterface { @Getter private NamespaceType namespaceType; @Getter - private AtomicBoolean commonCollection; + private AtomicBoolean catchAllEntity; @Getter - private String commonCollectionName; + private String catchAllEntityName; private final long databaseId; private final int userId; private final boolean ssl; // todo this. - boolean createCommonCollection = false; private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -187,26 +186,25 @@ public MqttStreamClient( TransactionManager transactionManager, Authenticator au this.namespaceName = name; this.namespaceType = type; - this.commonCollection = new AtomicBoolean( Boolean.parseBoolean( settings.get( "commonCollection" ) ) ); - this.commonCollectionName = settings.get( "commonCollectionName" ) == null ? - settings.get( "commonCollectionName" ) : settings.get( "commonCollectionName" ) + this.catchAllEntity = new AtomicBoolean( Boolean.parseBoolean( settings.get( "catchAllEntity" ) ) ); + this.catchAllEntityName = settings.get( "catchAllEntityName" ) == null ? + settings.get( "catchAllEntityName" ) : settings.get( "catchAllEntityName" ) .trim() .replace( '#', '_' ) .replace( '+', '_' ) .replace( '/', '_' ); - if ( this.commonCollection.get() ) { - if ( this.commonCollectionName == null || this.commonCollectionName.isEmpty() || this.commonCollectionName.isBlank() ) { - throw new NullPointerException( "commonCollection is set to true but no valid collection name was given! Please enter a collection name." ); - } else if ( !collectionExists( this.commonCollectionName ) ) { - this.createCommonCollection = true; - createStreamCollection( this.commonCollectionName ); + if ( this.catchAllEntity.get() ) { + if ( this.catchAllEntityName == null || this.catchAllEntityName.isEmpty() || this.catchAllEntityName.isBlank() ) { + throw new NullPointerException( "catchAllEntity is set to true but no valid collection name was given! Please enter a collection name." ); + } else if ( !collectionExists( this.catchAllEntityName ) ) { + createStreamCollection( this.catchAllEntityName ); } } else if ( settings.get( "topics" ) != null ) { for ( String topic : toList( settings.get( "topics" ) ) ) { topic = topic.replace( '#', '_' ) .replace( '+', '_' ) .replace( '/', '_' ); - if ( !this.commonCollection.get() && !collectionExists( topic ) ) { + if ( !this.catchAllEntity.get() && !collectionExists( topic ) ) { createStreamCollection( topic ); } } @@ -417,33 +415,33 @@ protected void reloadSettings( List updatedSettings ) { } } break; - case "commonCollection": - this.commonCollection.set( Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ) ); + case "catchAllEntity": + this.catchAllEntity.set( Boolean.parseBoolean( this.getCurrentSettings().get( "catchAllEntity" ) ) ); createAllCollections(); break; - case "commonCollectionName": - String newCommonCollectionName = this.getCurrentSettings().get( "commonCollectionName" ).trim(); - newCommonCollectionName = newCommonCollectionName == null ? null : newCommonCollectionName.trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + case "catchAllEntityName": + String newcatchAllEntityName = this.getCurrentSettings().get( "catchAllEntityName" ).trim(); + newcatchAllEntityName = newcatchAllEntityName == null ? null : newcatchAllEntityName.trim().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); boolean mode; - if ( updatedSettings.contains( "commonCollection" ) ) { - mode = Boolean.parseBoolean( this.getCurrentSettings().get( "commonCollection" ) ); + if ( updatedSettings.contains( "catchAllEntity" ) ) { + mode = Boolean.parseBoolean( this.getCurrentSettings().get( "catchAllEntity" ) ); } else { - mode = this.commonCollection.get(); + mode = this.catchAllEntity.get(); } if ( mode ) { - if ( !(newCommonCollectionName.equals( "null" ) || newCommonCollectionName.isEmpty() || newCommonCollectionName.isBlank()) ) { - if ( !collectionExists( newCommonCollectionName ) ) { - createStreamCollection( this.commonCollectionName ); + if ( !(newcatchAllEntityName.equals( "null" ) || newcatchAllEntityName.isEmpty() || newcatchAllEntityName.isBlank()) ) { + if ( !collectionExists( newcatchAllEntityName ) ) { + createStreamCollection( this.catchAllEntityName ); } - this.commonCollectionName = newCommonCollectionName; + this.catchAllEntityName = newcatchAllEntityName; createAllCollections(); } else { - this.settings.put( "commonCollectionName", this.commonCollectionName ); - throw new NullPointerException( "commonCollection is set to FALSE but no valid collection name was given! Please enter a collection name." ); + this.settings.put( "catchAllEntityName", this.catchAllEntityName ); + throw new NullPointerException( "catchAllEntity is set to FALSE but no valid collection name was given! Please enter a collection name." ); } } else { - this.commonCollectionName = newCommonCollectionName; + this.catchAllEntityName = newcatchAllEntityName; } break; case "filterQuery": @@ -469,7 +467,7 @@ protected void subscribe( List newTopics ) { * * @param topic the topic the client should subscribe to. */ - protected void subscribe( String topic ) { + private void subscribe( String topic ) { client.subscribeWith().topicFilter( topic ) .callback( this::processMsg ) .send() @@ -506,7 +504,7 @@ protected void unsubscribe( List topics ) { } - protected void unsubscribe( String topic ) { + private void unsubscribe( String topic ) { client.unsubscribeWith().topicFilter( topic ).send().whenComplete( ( unsub, throwable ) -> { if ( throwable != null ) { synchronized ( settingsLock ) { @@ -608,12 +606,12 @@ protected void processMsg( Mqtt5Publish subMsg ) { private void insertInEntity( MqttMessage mqttMessage, Transaction transaction ) { StoringMqttMessage storingMqttMessage; synchronized ( settingsLock ) { - if ( !this.commonCollection.get() ) { + if ( !this.catchAllEntity.get() ) { String collectionToBeSaved; collectionToBeSaved = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); } else { - storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.commonCollectionName ); + storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.catchAllEntityName ); } } StreamCapture streamCapture = new StreamCapture( transaction ); @@ -716,19 +714,19 @@ private void createStreamCollection( String collectionName ) { private void createAllCollections() { synchronized ( settingsLock ) { - if ( !this.commonCollection.get() ) { + if ( !this.catchAllEntity.get() ) { for ( String t : this.topicsMap.keySet() ) { if ( !collectionExists( t ) ) { createStreamCollection( t ); } } } else { - if ( !(this.commonCollectionName == null || this.commonCollectionName.equals( "" ) || this.commonCollectionName.isBlank()) ) { - if ( !collectionExists( this.commonCollectionName ) ) { - createStreamCollection( this.commonCollectionName ); + if ( !(this.catchAllEntityName == null || this.catchAllEntityName.equals( "" ) || this.catchAllEntityName.isBlank()) ) { + if ( !collectionExists( this.catchAllEntityName ) ) { + createStreamCollection( this.catchAllEntityName ); } } else { - throw new NullPointerException( "commonCollection is set to 'true' but no valid collection name was given! Please enter a collection name." ); + throw new NullPointerException( "catchAllEntity is set to 'true' but no valid collection name was given! Please enter a collection name." ); } } } @@ -736,6 +734,14 @@ private void createAllCollections() { } + protected void publish( String topic, String payload) { + client.publishWith() + .topic( topic ) + .payload( payload.getBytes() ) + .send(); + } + + private Transaction getTransaction() { try { return transactionManager.startTransaction( this.userId, this.databaseId, false, "MQTT Stream" ); @@ -945,11 +951,9 @@ public MonitoringPage() { im.addGroup( informationGroupPub ); msgButton = new InformationAction( informationGroupPub, "Publish", ( parameters ) -> { String end = "Message was published!"; + try { - client.publishWith() - .topic( parameters.get( "topic" ) ) - .payload( parameters.get( "payload" ).getBytes() ) - .send(); + publish( parameters.get( "topic" ), parameters.get( "payload" ) ); } catch ( IllegalArgumentException e ) { throw new RuntimeException( e ); } diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index 3fc8361326..29bf3adedc 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -59,8 +59,8 @@ public void resetSettings() { initialSettings.clear(); initialSettings.put( "brokerAddress", "localhost" ); initialSettings.put( "brokerPort", "1883" ); - initialSettings.put( "commonCollectionName", "testCollection" ); - initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "catchAllEntityName", "testCollection" ); + initialSettings.put( "catchAllEntity", "true" ); initialSettings.put( "namespace", "testNamespace" ); initialSettings.put( "namespaceType", "DOCUMENT" ); initialSettings.put( "topics", "" ); @@ -77,8 +77,8 @@ public void resetSettings() { initialSettings ); changedSettings.clear(); - changedSettings.put( "commonCollectionName", "testCollection" ); - changedSettings.put( "commonCollection", "true" ); + changedSettings.put( "catchAllEntityName", "testCollection" ); + changedSettings.put( "catchAllEntity", "true" ); changedSettings.put( "namespace", "testNamespace" ); //changedSettings.put( "namespaceType", "DOCUMENT"); changedSettings.put( "topics", "" ); @@ -225,64 +225,64 @@ public void reloadSettingsWrongTypeNamespaceTest() { @Test - public void reloadSettingsCommonCollectionToFalseTest() { - changedSettings.replace( "commonCollection", "false" ); + public void reloadSettingsCatchAllEntityToFalseTest() { + changedSettings.replace( "catchAllEntity", "false" ); client.updateSettings( changedSettings ); - assertFalse( client.getCommonCollection().get() ); + assertFalse( client.getCatchAllEntity().get() ); } @Test - public void reloadSettingsCommonCollectionToTrueTest() { - changedSettings.replace( "commonCollection", "true" ); + public void reloadSettingsCatchAllEntityToTrueTest() { + changedSettings.replace( "catchAllEntity", "true" ); client.updateSettings( changedSettings ); - assertTrue( client.getCommonCollection().get() ); + assertTrue( client.getCatchAllEntity().get() ); } @Test - public void reloadSettingsNewCommonCollectionNameTest() { - changedSettings.replace( "commonCollectionName", "buttonCollection" ); + public void reloadSettingsNewCatchAllEntityNameTest() { + changedSettings.replace( "catchAllEntityName", "buttonCollection" ); client.updateSettings( changedSettings ); - assertEquals( "buttonCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); + assertEquals( "buttonCollection", client.getCatchAllEntityName() ); + assertTrue( client.getCatchAllEntity().get() ); } @Test - public void reloadSettingsExistingCommonCollectionNameTest() { - changedSettings.replace( "commonCollectionName", "testCollection" ); + public void reloadSettingsExistingCatchAllEntityNameTest() { + changedSettings.replace( "catchAllEntityName", "testCollection" ); client.updateSettings( changedSettings ); - assertEquals( "testCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); + assertEquals( "testCollection", client.getCatchAllEntityName() ); + assertTrue( client.getCatchAllEntity().get() ); } @Test - public void reloadSettingsCommonCollectionAndCommonCollectionNameTest() { - // testing special case: commonCollection changed from false to true + commonCollectionName changes - changedSettings.replace( "commonCollection", "false" ); + public void reloadSettingsCatchAllEntityAndCatchAllEntityNameTest() { + // testing special case: catchAllEntity changed from false to true + catchAllEntityName changes + changedSettings.replace( "catchAllEntity", "false" ); client.updateSettings( changedSettings ); - changedSettings.replace( "commonCollectionName", "buttonCollection" ); - changedSettings.replace( "commonCollection", "true" ); + changedSettings.replace( "catchAllEntityName", "buttonCollection" ); + changedSettings.replace( "catchAllEntity", "true" ); client.updateSettings( changedSettings ); - assertEquals( "buttonCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); + assertEquals( "buttonCollection", client.getCatchAllEntityName() ); + assertTrue( client.getCatchAllEntity().get() ); } @Test(expected = NullPointerException.class) - public void reloadSettingsCommonCollectionAndCommonCollectionNameTest2() { - // testing special case: commonCollection changed from false to true + commonCollectionName changes - changedSettings.replace( "commonCollection", "false" ); + public void reloadSettingsCatchAllEntityAndCatchAllEntityNameTest2() { + // testing special case: catchAllEntity changed from false to true + catchAllEntityName changes + changedSettings.replace( "catchAllEntity", "false" ); client.updateSettings( changedSettings ); - changedSettings.replace( "commonCollectionName", " " ); - changedSettings.replace( "commonCollection", "true" ); + changedSettings.replace( "catchAllEntityName", " " ); + changedSettings.replace( "catchAllEntity", "true" ); client.updateSettings( changedSettings ); - assertEquals( "testCollection", client.getCommonCollectionName() ); - assertTrue( client.getCommonCollection().get() ); + assertEquals( "testCollection", client.getCatchAllEntityName() ); + assertTrue( client.getCatchAllEntity().get() ); } From 60ee769729a4aa418c328fe3c340431b5ea2fdcf Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 13:52:34 +0200 Subject: [PATCH 100/114] removed unnecessary code --- .../org/polypheny/db/mqtt/StreamCaptureTest.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java index cf1fc8d8d2..4d37a0c7a4 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java @@ -74,8 +74,6 @@ public void insertIntTest() { capture.insert( storingmsg ); // getting content with index 0 because there will only be one document matching to this query BsonDocument result = Helper.filter( "{\"payload\":25}" ).get( 0 ); - System.out.println( "int Test" ); - System.out.println( result.toString() ); assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); assertEquals( 25, result.get( "payload" ).asInt32().intValue() ); @@ -90,8 +88,7 @@ public void insertDoubleTest() { //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); BsonDocument result = Helper.filter( "{\"payload\":25.54}" ).get( 0 ); - System.out.println( "double Test" ); - System.out.println( result.toString() ); + assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); assertEquals( 25.54, result.get( "payload" ).asDouble().getValue(), 0.1 ); assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); @@ -118,8 +115,6 @@ public void insertBooleanTest() { //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); BsonDocument result = Helper.filter( "{\"payload\":true}" ).get( 0 ); - System.out.println( "bool Test" ); - System.out.println( result.toString() ); List collection = Helper.scanCollection(); assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); @@ -151,6 +146,7 @@ public void insertArrayTest() { //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); BsonDocument result = Helper.filter( "{\"payload\":[1, 2, 3]}" ).get( 0 ); + List collection = Helper.scanCollection(); assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); BsonArray expectedPayload = new BsonArray(); expectedPayload.add( 0, new BsonInt32( 1 ) ); @@ -170,14 +166,12 @@ public void isIntTest() { @Test public void isDoubleTest() { - //StreamCapture capture = new StreamCapture( transaction ); assertTrue( capture.isDouble( "1.0" ) ); } @Test public void isBooleanTest() { - //StreamCapture capture = new StreamCapture( transaction ); assertTrue( capture.isBoolean( "false" ) ); } @@ -214,7 +208,6 @@ private static void createCollection() { } - //TODO: StreamCapture capture private static List scanCollection() { String sqlCollectionName = "testSpace" + "." + "testCollection"; Statement statement = transaction.createStatement(); @@ -232,7 +225,6 @@ private static List scanCollection() { } - //TODO StreamCapture capture private static List filter( String query ) { Statement statement = transaction.createStatement(); QueryParameters parameters = new MqlQueryParameters( String.format( "db.%s.find(%s)", "testCollection", query ), "testSpace", NamespaceType.DOCUMENT ); From e834f56fe491fe6b59baa6540578127639efa59e Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 13:52:56 +0200 Subject: [PATCH 101/114] Tests for sub + wildcards and unsub --- .../db/mqtt/MqttClientBrokerTest.java | 113 ++++++++++++++++-- 1 file changed, 101 insertions(+), 12 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java index ea6dcf9988..9aba791aed 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java @@ -18,10 +18,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicLong; import org.junit.Before; import org.junit.BeforeClass; @@ -50,8 +53,8 @@ public static void init() { initialSettings.clear(); initialSettings.put( "brokerAddress", "1883" ); initialSettings.put( "brokerPort", "1883" ); - initialSettings.put( "commonCollectionName", "testCollection" ); - initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "catchAllEntityName", "testCollection" ); + initialSettings.put( "catchAllEntity", "true" ); initialSettings.put( "namespace", "testNamespace" ); initialSettings.put( "namespaceType", "DOCUMENT" ); initialSettings.put( "topics", "button" ); @@ -69,14 +72,14 @@ public static void init() { } -/* @Before public void resetSettings() { + initialSettings.clear(); initialSettings.put( "brokerAddress", "1883" ); initialSettings.put( "brokerPort", "1883" ); - initialSettings.put( "commonCollectionName", "testCollection" ); - initialSettings.put( "commonCollection", "true" ); + initialSettings.put( "catchAllEntityName", "testCollection" ); + initialSettings.put( "catchAllEntity", "true" ); initialSettings.put( "namespace", "testNamespace" ); initialSettings.put( "namespaceType", "DOCUMENT" ); initialSettings.put( "topics", "" ); @@ -93,22 +96,108 @@ public void resetSettings() { initialSettings ); changedSettings.clear(); - changedSettings.put( "commonCollectionName", "testCollection" ); - changedSettings.put( "commonCollection", "true" ); + changedSettings.put( "catchAllEntityName", "testCollection" ); + changedSettings.put( "catchAllEntity", "true" ); changedSettings.put( "namespace", "testNamespace" ); changedSettings.put( "topics", "" ); changedSettings.put( "filterQuery", "" ); } - */ + + + private void simulateIoTDevices() { + client.publish( "device1/online", "true" ); + client.publish( "device1/sensor/measurements", "[28,30,35 ]" ); + client.publish( "device1/sensor/measurements/unit", "C" ); + client.publish( "device1/sensor/battery", "86" ); + + client.publish( "device2/online", "true" ); + client.publish( "device2/location/info", "Basel" ); + client.publish( "device2/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device2\"}" ); + } + + + @Test + public void simpleSubscribeUnsubscribeTest() { + changedSettings.replace( "topics", "device1/sensor/battery" ); + //All subscribed topics so far are unsubscribed + client.updateSettings( changedSettings ); + assertEquals( 1, client.getTopicsMap().size() ); + assertEquals( 0, client.getTopicsMap().get( "device1/sensor/battery" ).intValue() ); + simulateIoTDevices(); + assertEquals( 1, client.getTopicsMap().get( "device1/sensor/battery" ).intValue() ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); + } + + + @Test + public void subscribeWithWildcardHashtagTest() { + changedSettings.replace( "topics", "#" ); + client.updateSettings( changedSettings ); + simulateIoTDevices(); + assertEquals( 7, client.getTopicsMap().get( "#" ).intValue() ); + } + + + @Test + public void subscribeWithWildcardHashtagAtEndTest() { + changedSettings.replace( "topics", "device1/#" ); + client.updateSettings( changedSettings ); + simulateIoTDevices(); + assertEquals( 4, client.getTopicsMap().get( "device1/#" ).intValue() ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/online", "true" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device1\"}" } ) ); + } + + + @Test + public void subscribeWithWildcardPlusAtEndTest() { + changedSettings.replace( "topics", "device1/sensor/+" ); + client.updateSettings( changedSettings ); + simulateIoTDevices(); + assertEquals( 3, client.getTopicsMap().get( "device1/sensor/+" ).intValue() ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device1\"}" } ) ); + } + + + @Test + public void subscribeWithWildcardPlusInMiddleTest() { + changedSettings.replace( "topics", "device2/+/info" ); + client.updateSettings( changedSettings ); + simulateIoTDevices(); + assertEquals( 2, client.getTopicsMap().get( "device2/+/info" ).intValue() ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device2/location/info", "Basel" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device2/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device2\"}" } ) ); + + } + @Test - public void connectionTest() { - Map topicsMap = client.getTopicsMap(); - assertEquals( 1, topicsMap.size() ); - assertEquals( 0, topicsMap.get( "button" ) ); + public void subscribeWithWildcardPlusAtBeginningTest() { + changedSettings.replace( "topics", "+/online" ); + client.updateSettings( changedSettings ); + simulateIoTDevices(); + assertEquals( 2, client.getTopicsMap().get( "+/online" ).intValue() ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/online", "true" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device2/online", "true" } ) ); } +// +// subscribe to topic +// + wildcards + +// richtige query für wildcard topic holen +// collections werden zu beginn richtig erstellt +// catchAllEntity +// collectionPerTopic +// 1 msg handling test -> verify dass methoden aufgerufen werden + + + } From 64c8bcd202a4f6cea7a582ae3d1ca70261677454 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 15:58:03 +0200 Subject: [PATCH 102/114] removed methods related to ssl --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 167 +----------------- 1 file changed, 2 insertions(+), 165 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index ca317e87f4..0539c560c6 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -161,7 +161,6 @@ public static class MqttStreamClient extends QueryInterface { private String catchAllEntityName; private final long databaseId; private final int userId; - private final boolean ssl; // todo this. private final Object settingsLock = new Object(); private final MonitoringPage monitoringPage; @@ -214,33 +213,18 @@ public MqttStreamClient( TransactionManager transactionManager, Authenticator au saveQueriesInMap( queryString ); } - this.ssl = Boolean.parseBoolean( settings.get( "Tsl/SslConnection" ) ); - } @Override public void run() { - if ( ssl ) { - this.client = MqttClient.builder().useMqttVersion5() - .identifier( getUniqueName() ) - .serverHost( brokerAddress ) - .serverPort( brokerPort ) - .automaticReconnectWithDefaultConfig() - .sslConfig() - //TODO: delete or enter password from GUI password thinghere and in method - .keyManagerFactory( SslHelper.createKeyManagerFactory( "polyphenyClient.crt", "polyphenyClient.key", "" ) ) - .trustManagerFactory( SslHelper.createTrustManagerFactory( "ca.crt" ) ) - .applySslConfig() - .buildAsync(); - } else { + this.client = MqttClient.builder().useMqttVersion5() .identifier( getUniqueName() ) .serverHost( brokerAddress ) .serverPort( brokerPort ) .automaticReconnectWithDefaultConfig() .buildAsync(); - } client.connectWith().send().whenComplete( ( connAck, throwable ) -> { if ( throwable != null ) { @@ -473,7 +457,7 @@ private void subscribe( String topic ) { .send() .whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { - //TODO: change settings correctly: + //TODO: change settings correctly: Test this List topicsList = toList( this.getCurrentSettings().get( "topics" ) ); StringBuilder stringBuilder = new StringBuilder(); for ( String t : topicsList ) { @@ -763,151 +747,6 @@ public String getInterfaceType() { } - private static class SslHelper { - - private static KeyFactory getKeyFactoryInstance() { - try { - return KeyFactory.getInstance( "RSA" ); - } catch ( NoSuchAlgorithmException e ) { - throw new RuntimeException( e ); - } - } - - - private static X509Certificate createX509CertificateFromFile( final String certificateFileName ) { - String path = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + certificateFileName; - final File file = PolyphenyHomeDirManager.getInstance().getFileIfExists( path ); - if ( !file.isFile() ) { - throw new RuntimeException( String.format( "The certificate file %s doesn't exist.", certificateFileName ) ); - } - final X509Certificate certificate; - try { - final CertificateFactory certificateFactoryX509 = CertificateFactory.getInstance( "X.509" ); - final InputStream inputStream = new FileInputStream( file ); - certificate = (X509Certificate) certificateFactoryX509.generateCertificate( inputStream ); - inputStream.close(); - } catch ( Exception e ) { - throw new RuntimeException( e ); - } - - return certificate; - } - - - /* - private PrivateKey createPrivateKeyFromPemFile(final String keyFileName) { - final PrivateKey privateKey; - try { - final PemReader pemReader = new PemReader( new FileReader( keyFileName ) ); - final PemObject pemObject = pemReader.readPemObject(); - final byte[] pemContent = pemObject.getContent(); - pemReader.close(); - final PKCS8EncodedKeySpec encodedKeySpec = new PKCS8EncodedKeySpec( pemContent ); - final KeyFactory keyFactory = getKeyFactoryInstance(); - privateKey = keyFactory.generatePrivate( encodedKeySpec ); - } catch ( Exception e ) { - throw new RuntimeException(e); - } - return privateKey; - } - */ - private static KeyManagerFactory createKeyManagerFactory( String clientCertificateFileName, String clientKeyFileName, final String clientKeyPassword ) { - final KeyManagerFactory keyManagerFactory; - try { - //todo: this.getUniqueName() in path - String clientCrtPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.p12"; - File crtFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( clientCrtPath ); - //final X509Certificate clientCertificate = createX509CertificateFromFile("polyphenyClient.p12"); - InputStream crtStream = new FileInputStream( crtFile ); - final KeyStore ks = KeyStore.getInstance( "PKCS12" ); - ks.load( crtStream, "1234".toCharArray() ); - X509Certificate clientCertificate = (X509Certificate) ks.getCertificate( "alias" ); - keyManagerFactory = KeyManagerFactory.getInstance( KeyManagerFactory.getDefaultAlgorithm() ); - keyManagerFactory.init( ks, "".toCharArray() ); - /* - // load client certificate: - - - //CertificateFactory cf = CertificateFactory.getInstance("X.509"); - //Certificate clientCertificate = cf.generateCertificate( crtStream ); - crtStream.close();*/ - - // - //ks.load( null, null ); - //Certificate clientCertificate = ks.getCertificate( "alias" ); - - //load client key: - String keyPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "polyphenyClient.key"; - File keyFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( keyPath ); - InputStream keyStream = new FileInputStream( keyFile ); - /* - String key = keyStream.toString(); - key = key.replace( "-----BEGIN PRIVATE KEY-----", "" ).replace( "-----END PRIVATE KEY-----", "" ); - - */ - - - - - /* - - - // create private key method:------------------------------------------ - final PemReader pemReader = new PemReader(new FileReader(keyFile)); - final PemObject pemObject = pemReader.readPemObject(); - final byte[] pemContent = pemObject.getContent(); - pemReader.close(); - final PKCS8EncodedKeySpec encodedKeySpec = new PKCS8EncodedKeySpec(pemContent); - final KeyFactory keyFactory = getKeyFactoryInstance(); - final PrivateKey privateKey = keyFactory.generatePrivate(encodedKeySpec); - - //--------------------------------------------------------------------- - - - // create keystore - final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - keyStore.load(null, null); - keyStore.setCertificateEntry("certificate", clientCertificate); - keyStore.setKeyEntry("private-key", privateKey, - "".toCharArray(), - new Certificate[] { clientCertificate }); - keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, "".toCharArray()); - */ - } catch ( Exception e ) { - throw new RuntimeException( e ); - } - return keyManagerFactory; - } - - - private static TrustManagerFactory createTrustManagerFactory( final String caCertificateFileName ) { - final TrustManagerFactory trustManagerFactory; - try { - // load ca certificate: - //todo: this.getUniqueName() in path - /* - String caPath = "certs" + File.separator + "mqttStreamPlugin" + File.separator + "mosquitto" + File.separator + "ca.crt"; - File caFile = PolyphenyHomeDirManager.getInstance().getFileIfExists( caPath ); - InputStream ca = new FileInputStream( caFile ); - CertificateFactory cf = CertificateFactory.getInstance( "X.509" ); - Certificate caCertificate = cf.generateCertificate( ca ); - ca.close(); -*/ - final X509Certificate caCertificate = (X509Certificate) createX509CertificateFromFile( caCertificateFileName ); - final KeyStore trustStore = KeyStore.getInstance( KeyStore.getDefaultType() ); - trustStore.load( null, null ); - trustStore.setCertificateEntry( "ca-certificate", caCertificate ); - trustManagerFactory = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); - trustManagerFactory.init( trustStore ); - } catch ( Exception e ) { - throw new RuntimeException( e ); - } - return trustManagerFactory; - } - - } - private class MonitoringPage { @@ -1007,8 +846,6 @@ public void update() { brokerKv.putPair( "Broker version of MQTT", client.getConfig().getMqttVersion() + "" ); brokerKv.putPair( "Client state", client.getState() + "" ); brokerKv.putPair( "Client identifier", client.getConfig().getClientIdentifier().get() + "" ); - //TODO: check this after having SSL Configuration. - brokerKv.putPair( "SSL configuration", client.getConfig().getSslConfig() + "" ); } From 9b9a5706fc81607a1e230164abc9d8c08789a9d2 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 15:58:45 +0200 Subject: [PATCH 103/114] inserted tests for verifying collection creation --- .../db/mqtt/MqttStreamClientTest.java | 50 ++++++++++++++++--- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java index 29bf3adedc..e066fb3eca 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttStreamClientTest.java @@ -29,6 +29,12 @@ import org.junit.BeforeClass; import org.junit.Test; import org.polypheny.db.TestHelper; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.NamespaceType; +import org.polypheny.db.catalog.Catalog.Pattern; +import org.polypheny.db.catalog.entity.CatalogCollection; +import org.polypheny.db.catalog.entity.CatalogSchema; +import org.polypheny.db.catalog.exceptions.UnknownSchemaException; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamClient; @@ -147,6 +153,9 @@ public void saveNestedQueryTest() { } + + + @Test public void toListEmptyTest() { List result = client.toList( "" ); @@ -224,19 +233,16 @@ public void reloadSettingsWrongTypeNamespaceTest() { } - @Test - public void reloadSettingsCatchAllEntityToFalseTest() { - changedSettings.replace( "catchAllEntity", "false" ); - client.updateSettings( changedSettings ); - assertFalse( client.getCatchAllEntity().get() ); - } - - @Test public void reloadSettingsCatchAllEntityToTrueTest() { changedSettings.replace( "catchAllEntity", "true" ); client.updateSettings( changedSettings ); assertTrue( client.getCatchAllEntity().get() ); + + Catalog catalog = Catalog.getInstance(); + Pattern pattern = new Pattern( client.getCatchAllEntityName() ); + List collectionList = catalog.getCollections( Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern ); + assertFalse( collectionList.isEmpty() ); } @@ -246,6 +252,11 @@ public void reloadSettingsNewCatchAllEntityNameTest() { client.updateSettings( changedSettings ); assertEquals( "buttonCollection", client.getCatchAllEntityName() ); assertTrue( client.getCatchAllEntity().get() ); + + Catalog catalog = Catalog.getInstance(); + Pattern pattern = new Pattern( client.getCatchAllEntityName() ); + List collectionList = catalog.getCollections( Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern ); + assertFalse( collectionList.isEmpty() ); } @@ -269,6 +280,11 @@ public void reloadSettingsCatchAllEntityAndCatchAllEntityNameTest() { client.updateSettings( changedSettings ); assertEquals( "buttonCollection", client.getCatchAllEntityName() ); assertTrue( client.getCatchAllEntity().get() ); + + Catalog catalog = Catalog.getInstance(); + Pattern pattern = new Pattern( "buttonCollection" ); + List collectionList = catalog.getCollections( Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern ); + assertFalse( collectionList.isEmpty() ); } @@ -285,5 +301,23 @@ public void reloadSettingsCatchAllEntityAndCatchAllEntityNameTest2() { assertTrue( client.getCatchAllEntity().get() ); } + static class Helper { + static long getExistingNamespaceId( String namespaceName, NamespaceType namespaceType ) { + Catalog catalog = Catalog.getInstance(); + CatalogSchema schema; + try { + schema = catalog.getSchema( Catalog.defaultDatabaseId, namespaceName ); + } catch ( UnknownSchemaException e ) { + throw new RuntimeException( e ); + } + assert schema != null; + if ( schema.namespaceType == namespaceType ) { + return schema.id; + } else { + throw new RuntimeException( "There is already a namespace existing in this database with the given name but of another type. Please change the namespace name or the type." ); + } + } + } + } From 38067a582a02f7fa4ce55e21a3e70ff9146749ab Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 16:00:49 +0200 Subject: [PATCH 104/114] added getWildcard and catchAllEntity=false tests --- .../db/mqtt/MqttClientBrokerTest.java | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java index 9aba791aed..9ba681d913 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java @@ -30,8 +30,12 @@ import org.junit.BeforeClass; import org.junit.Test; import org.polypheny.db.TestHelper; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.Catalog.Pattern; +import org.polypheny.db.catalog.entity.CatalogCollection; import org.polypheny.db.iface.QueryInterface; import org.polypheny.db.iface.QueryInterfaceManager; +import org.polypheny.db.mqtt.MqttStreamClientTest.Helper; import org.polypheny.db.mqtt.MqttStreamPlugin.MqttStreamClient; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; @@ -51,7 +55,7 @@ public static void init() { transactionManager = testHelper.getTransactionManager(); transaction = testHelper.getTransaction(); initialSettings.clear(); - initialSettings.put( "brokerAddress", "1883" ); + initialSettings.put( "brokerAddress", "localhost" ); initialSettings.put( "brokerPort", "1883" ); initialSettings.put( "catchAllEntityName", "testCollection" ); initialSettings.put( "catchAllEntity", "true" ); @@ -147,7 +151,7 @@ public void subscribeWithWildcardHashtagAtEndTest() { assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); assertTrue( client.getMessageQueue().contains( new String[]{ "device1/online", "true" } ) ); assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device1\"}" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements/unit", "C" } ) ); } @@ -159,7 +163,7 @@ public void subscribeWithWildcardPlusAtEndTest() { assertEquals( 3, client.getTopicsMap().get( "device1/sensor/+" ).intValue() ); assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device1\"}" } ) ); + assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements/unit", "C" } ) ); } @@ -186,18 +190,49 @@ public void subscribeWithWildcardPlusAtBeginningTest() { } -// -// subscribe to topic -// + wildcards + @Test + public void reloadSettingsCatchAllEntityToFalseTest() { -// richtige query für wildcard topic holen -// collections werden zu beginn richtig erstellt -// catchAllEntity -// collectionPerTopic -// 1 msg handling test -> verify dass methoden aufgerufen werden + changedSettings.replace( "topics", "device1/online, device2/+/info, device1/sensor/#" ); + client.updateSettings( changedSettings ); + changedSettings.replace( "catchAllEntity", "false" ); + client.updateSettings( changedSettings ); + assertFalse( client.getCatchAllEntity().get() ); + + Catalog catalog1 = Catalog.getInstance(); + Pattern pattern1 = new Pattern( "device1_online" ); + List collectionList1 = catalog1.getCollections( MqttStreamClientTest.Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern1 ); + assertFalse( collectionList1.isEmpty() ); + + Catalog catalog2 = Catalog.getInstance(); + Pattern pattern2 = new Pattern( "device2___info" ); + List collectionList2 = catalog2.getCollections( Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern2 ); + assertFalse( collectionList2.isEmpty() ); + + Catalog catalog3 = Catalog.getInstance(); + Pattern pattern3 = new Pattern( "device1_sensor__" ); + List collectionList3 = catalog3.getCollections( Helper.getExistingNamespaceId( client.getNamespaceName(), client.getNamespaceType() ), pattern3 ); + assertFalse( collectionList3.isEmpty() ); + } + @Test + public void getWildcardTopicWithHashtagTest() { + changedSettings.replace( "topics", "device1/sensor/#"); + changedSettings.replace( "filterQuery", "device1/sensor/#:{}" ); + client.updateSettings( changedSettings ); + String resultWildcardTopic1 = client.getWildcardTopic( "device1/sensor/measurements/unit" ); + assertEquals( "device1/sensor/#", resultWildcardTopic1 ); + } + @Test + public void getWildcardTopicWithPlusTest() { + changedSettings.replace( "topics", "+/online"); + changedSettings.replace( "filterQuery", "+/online:{\"$$ROOT\":false}" ); + client.updateSettings( changedSettings ); + String resultWildcardTopic = client.getWildcardTopic( "device1/online" ); + assertEquals( "+/online", resultWildcardTopic ); + } } From a1bc339b0d31d28e0eb07c84dd74baef85c227ca Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 16:53:50 +0200 Subject: [PATCH 105/114] renamed methods and vars used entity instead of collection (only where it is necessary) --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 165 +++++++++--------- 1 file changed, 81 insertions(+), 84 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 0539c560c6..84235516da 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -21,15 +21,7 @@ import com.hivemq.client.mqtt.MqttClient; import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient; import com.hivemq.client.mqtt.mqtt5.message.publish.Mqtt5Publish; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; import java.nio.charset.Charset; -import java.security.KeyFactory; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -40,8 +32,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; @@ -75,7 +65,6 @@ import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; -import org.polypheny.db.util.PolyphenyHomeDirManager; public class MqttStreamPlugin extends Plugin { @@ -131,8 +120,7 @@ public static class MqttStreamClient extends QueryInterface { new QueryInterfaceSettingList( "catchAllEntity", false, true, true, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ), new QueryInterfaceSettingString( "catchAllEntityName", true, false, true, null ), new QueryInterfaceSettingString( "topics", false, true, true, null ), - new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ), - new QueryInterfaceSettingList( "Tsl/SslConnection", false, true, false, new ArrayList<>( List.of( "TRUE", "FALSE" ) ) ) ); + new QueryInterfaceSettingString( "filterQuery", true, false, true, "" ) ); @Getter private final String brokerAddress; @@ -194,17 +182,17 @@ public MqttStreamClient( TransactionManager transactionManager, Authenticator au .replace( '/', '_' ); if ( this.catchAllEntity.get() ) { if ( this.catchAllEntityName == null || this.catchAllEntityName.isEmpty() || this.catchAllEntityName.isBlank() ) { - throw new NullPointerException( "catchAllEntity is set to true but no valid collection name was given! Please enter a collection name." ); - } else if ( !collectionExists( this.catchAllEntityName ) ) { - createStreamCollection( this.catchAllEntityName ); + throw new NullPointerException( "catchAllEntity is set to true but no valid entity name was given! Please enter a entity name." ); + } else if ( !entityExists( this.catchAllEntityName ) ) { + createEntity( this.catchAllEntityName ); } } else if ( settings.get( "topics" ) != null ) { for ( String topic : toList( settings.get( "topics" ) ) ) { topic = topic.replace( '#', '_' ) .replace( '+', '_' ) .replace( '/', '_' ); - if ( !this.catchAllEntity.get() && !collectionExists( topic ) ) { - createStreamCollection( topic ); + if ( !this.catchAllEntity.get() && !entityExists( topic ) ) { + createEntity( topic ); } } } @@ -219,12 +207,12 @@ public MqttStreamClient( TransactionManager transactionManager, Authenticator au @Override public void run() { - this.client = MqttClient.builder().useMqttVersion5() - .identifier( getUniqueName() ) - .serverHost( brokerAddress ) - .serverPort( brokerPort ) - .automaticReconnectWithDefaultConfig() - .buildAsync(); + this.client = MqttClient.builder().useMqttVersion5() + .identifier( getUniqueName() ) + .serverHost( brokerAddress ) + .serverPort( brokerPort ) + .automaticReconnectWithDefaultConfig() + .buildAsync(); client.connectWith().send().whenComplete( ( connAck, throwable ) -> { if ( throwable != null ) { @@ -348,7 +336,7 @@ protected void reloadSettings( List updatedSettings ) { } this.namespaceName = newNamespaceName; this.namespaceType = type; - createAllCollections(); + createAllEntities(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); @@ -361,7 +349,7 @@ protected void reloadSettings( List updatedSettings ) { createNamespace( newNamespaceName, this.namespaceType ); } this.namespaceName = newNamespaceName; - createAllCollections(); + createAllEntities(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); throw new RuntimeException( e ); @@ -379,7 +367,7 @@ protected void reloadSettings( List updatedSettings ) { } this.namespaceName = newName; this.namespaceType = newNamespaceType; - createAllCollections(); + createAllEntities(); } catch ( RuntimeException e ) { this.settings.put( "namespace", this.namespaceName ); this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); @@ -392,7 +380,7 @@ protected void reloadSettings( List updatedSettings ) { createNamespace( this.namespaceName, newNamespaceType ); } this.namespaceType = newNamespaceType; - createAllCollections(); + createAllEntities(); } catch ( RuntimeException e ) { this.settings.put( "namespaceType", String.valueOf( this.namespaceType ) ); throw new RuntimeException( e ); @@ -401,7 +389,7 @@ protected void reloadSettings( List updatedSettings ) { break; case "catchAllEntity": this.catchAllEntity.set( Boolean.parseBoolean( this.getCurrentSettings().get( "catchAllEntity" ) ) ); - createAllCollections(); + createAllEntities(); break; case "catchAllEntityName": String newcatchAllEntityName = this.getCurrentSettings().get( "catchAllEntityName" ).trim(); @@ -414,14 +402,14 @@ protected void reloadSettings( List updatedSettings ) { } if ( mode ) { if ( !(newcatchAllEntityName.equals( "null" ) || newcatchAllEntityName.isEmpty() || newcatchAllEntityName.isBlank()) ) { - if ( !collectionExists( newcatchAllEntityName ) ) { - createStreamCollection( this.catchAllEntityName ); + if ( !entityExists( newcatchAllEntityName ) ) { + createEntity( this.catchAllEntityName ); } this.catchAllEntityName = newcatchAllEntityName; - createAllCollections(); + createAllEntities(); } else { this.settings.put( "catchAllEntityName", this.catchAllEntityName ); - throw new NullPointerException( "catchAllEntity is set to FALSE but no valid collection name was given! Please enter a collection name." ); + throw new NullPointerException( "catchAllEntity is set to FALSE but no valid entity name was given! Please enter a entity name." ); } } else { @@ -591,9 +579,9 @@ private void insertInEntity( MqttMessage mqttMessage, Transaction transaction ) StoringMqttMessage storingMqttMessage; synchronized ( settingsLock ) { if ( !this.catchAllEntity.get() ) { - String collectionToBeSaved; - collectionToBeSaved = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); - storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, collectionToBeSaved ); + String entityName; + entityName = mqttMessage.getTopic().replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, entityName ); } else { storingMqttMessage = new StoringMqttMessage( mqttMessage, this.namespaceName, this.namespaceType, getUniqueName(), this.databaseId, this.userId, this.catchAllEntityName ); } @@ -655,70 +643,80 @@ protected List toList( String string ) { /** - * @param collectionName - * @return true: collection already exists, false: collection does not exist. + * @param entityName + * @return true: entity already exists, false: entity does not exist. */ - private boolean collectionExists( String collectionName ) { - collectionName = collectionName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); - Catalog catalog = Catalog.getInstance(); - Pattern pattern = new Pattern( collectionName ); - List collectionList = null; - synchronized ( settingsLock ) { - collectionList = catalog.getCollections( getNamespaceId( this.namespaceName, this.namespaceType ), pattern ); + private boolean entityExists( String entityName ) { + if ( this.namespaceType == NamespaceType.DOCUMENT ) { + String collectionName = entityName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + Catalog catalog = Catalog.getInstance(); + Pattern pattern = new Pattern( collectionName ); + List collectionList = null; + synchronized ( settingsLock ) { + collectionList = catalog.getCollections( getNamespaceId( this.namespaceName, this.namespaceType ), pattern ); + } + return !collectionList.isEmpty(); + } else { + // handle other namespace types + return false; } - return !collectionList.isEmpty(); } - private void createStreamCollection( String collectionName ) { - collectionName = collectionName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); - Transaction transaction = getTransaction(); - Statement statement = transaction.createStatement(); - long namespaceID; - synchronized ( settingsLock ) { - namespaceID = getNamespaceId( this.namespaceName, this.namespaceType ); - } - try { - List dataStores = new ArrayList<>(); - DdlManager.getInstance().createCollection( - namespaceID, - collectionName, - true, //only creates collection if it does not already exist. - dataStores.size() == 0 ? null : dataStores, - PlacementType.MANUAL, - statement ); - transaction.commit(); - } catch ( EntityAlreadyExistsException | TransactionException e ) { - throw new RuntimeException( "Error while creating a new collection:", e ); - } catch ( UnknownSchemaIdRuntimeException e3 ) { - throw new RuntimeException( "New collection could not be created.", e3 ); + private void createEntity( String entityName ) { + if ( this.namespaceType == NamespaceType.DOCUMENT ) { + String collectionName = entityName.replace( '#', '_' ).replace( '+', '_' ).replace( '/', '_' ); + Transaction transaction = getTransaction(); + Statement statement = transaction.createStatement(); + long namespaceID; + synchronized ( settingsLock ) { + namespaceID = getNamespaceId( this.namespaceName, this.namespaceType ); + } + try { + List dataStores = new ArrayList<>(); + DdlManager.getInstance().createCollection( + namespaceID, + collectionName, + true, //only creates collection if it does not already exist. + dataStores.size() == 0 ? null : dataStores, + PlacementType.MANUAL, + statement ); + transaction.commit(); + } catch ( EntityAlreadyExistsException | TransactionException e ) { + throw new RuntimeException( "Error while creating a new collection:", e ); + } catch ( UnknownSchemaIdRuntimeException e3 ) { + throw new RuntimeException( "New collection could not be created.", e3 ); + } } } - private void createAllCollections() { - synchronized ( settingsLock ) { - if ( !this.catchAllEntity.get() ) { - for ( String t : this.topicsMap.keySet() ) { - if ( !collectionExists( t ) ) { - createStreamCollection( t ); - } - } - } else { - if ( !(this.catchAllEntityName == null || this.catchAllEntityName.equals( "" ) || this.catchAllEntityName.isBlank()) ) { - if ( !collectionExists( this.catchAllEntityName ) ) { - createStreamCollection( this.catchAllEntityName ); + private void createAllEntities() { + if ( this.namespaceType == NamespaceType.DOCUMENT ) { + synchronized ( settingsLock ) { + if ( !this.catchAllEntity.get() ) { + for ( String t : this.topicsMap.keySet() ) { + if ( !entityExists( t ) ) { + createEntity( t ); + } } } else { - throw new NullPointerException( "catchAllEntity is set to 'true' but no valid collection name was given! Please enter a collection name." ); + if ( !(this.catchAllEntityName == null || this.catchAllEntityName.equals( "" ) || this.catchAllEntityName.isBlank()) ) { + if ( !entityExists( this.catchAllEntityName ) ) { + createEntity( this.catchAllEntityName ); + } + } else { + throw new NullPointerException( "catchAllEntity is set to 'true' but no valid entity name was given! Please enter a entity name." ); + } } } + } else { + // handle other namespace types } - } - protected void publish( String topic, String payload) { + protected void publish( String topic, String payload ) { client.publishWith() .topic( topic ) .payload( payload.getBytes() ) @@ -747,7 +745,6 @@ public String getInterfaceType() { } - private class MonitoringPage { private final InformationPage informationPage; From 68431a76c687756d085a4e1239e73ade43ccebef Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 16:57:22 +0200 Subject: [PATCH 106/114] inserted query for namespace type --- .../org/polypheny/db/mqtt/StreamCapture.java | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 3c4774d1e4..9c9dd5a52d 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -29,6 +29,7 @@ import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.prepare.Context; import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.transaction.Statement; @@ -56,43 +57,45 @@ public void insert( StoringMqttMessage storingMqttMessage ) { private void insertMessage() { - String sqlCollectionName = this.storingMqttMessage.getNamespaceName() + "." + this.storingMqttMessage.getEntityName(); - Statement statement = transaction.createStatement(); - - // Builder which allows to construct the algebra tree which is equivalent to query and is executed - AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); - - BsonDocument document = new BsonDocument(); - document.put( "source", new BsonString( this.storingMqttMessage.getUniqueNameOfInterface() ) ); - document.put( "topic", new BsonString( this.storingMqttMessage.getTopic() ) ); - String msg = this.storingMqttMessage.getMessage(); - BsonValue value; - if ( msg.contains( "{" ) && msg.contains( "}" ) ) { - value = BsonDocument.parse( msg ); - } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { - BsonArray bsonArray = new BsonArray(); - msg = msg.replace( "[", "" ).replace( "]", "" ); - String[] msglist = msg.split( "," ); - for ( String stringValue : msglist ) { - stringValue = stringValue.trim(); - bsonArray.add( getBsonValue( stringValue ) ); + if ( this.storingMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { + String sqlCollectionName = this.storingMqttMessage.getNamespaceName() + "." + this.storingMqttMessage.getEntityName(); + Statement statement = transaction.createStatement(); + + // Builder which allows to construct the algebra tree which is equivalent to query and is executed + AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); + + BsonDocument document = new BsonDocument(); + document.put( "source", new BsonString( this.storingMqttMessage.getUniqueNameOfInterface() ) ); + document.put( "topic", new BsonString( this.storingMqttMessage.getTopic() ) ); + String msg = this.storingMqttMessage.getMessage(); + BsonValue value; + if ( msg.contains( "{" ) && msg.contains( "}" ) ) { + value = BsonDocument.parse( msg ); + } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { + BsonArray bsonArray = new BsonArray(); + msg = msg.replace( "[", "" ).replace( "]", "" ); + String[] msglist = msg.split( "," ); + for ( String stringValue : msglist ) { + stringValue = stringValue.trim(); + bsonArray.add( getBsonValue( stringValue ) ); + } + value = bsonArray; + } else { + // msg is a single value + value = getBsonValue( msg ); } - value = bsonArray; - } else { - // msg is a single value - value = getBsonValue( msg ); - } - document.put( "payload", value ); + document.put( "payload", value ); - AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); + AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); - AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); - // for inserts and all DML queries only a number is returned - List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); - try { - transaction.commit(); - } catch ( TransactionException e ) { - throw new RuntimeException( e ); + AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); + // for inserts and all DML queries only a number is returned + List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); + try { + transaction.commit(); + } catch ( TransactionException e ) { + throw new RuntimeException( e ); + } } } From 6b100d9242be3d9b6f41650d3336b6821c6a9677 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 18:50:47 +0200 Subject: [PATCH 107/114] fixed dependency problem --- plugins/mqtt-stream/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index a3546f4fe7..cd3f63f88f 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -75,6 +75,7 @@ compileJava { dependsOn(":core:processResources") dependsOn(":information:processResources") dependsOn(":monitoring:processResources") + dependsOn(":plugins:mql-language:processResources") } delombok { From 9a34ee812072152126473a19de5310d2cac80705 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 18:51:36 +0200 Subject: [PATCH 108/114] minor changes --- .../polypheny/db/mqtt/StreamCaptureTest.java | 31 +++---------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java index 4d37a0c7a4..650b3f8216 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/StreamCaptureTest.java @@ -28,9 +28,7 @@ import org.junit.Test; import org.polypheny.db.TestHelper; import org.polypheny.db.adapter.DataStore; -import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; -import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.Catalog.NamespaceType; import org.polypheny.db.catalog.Catalog.PlacementType; @@ -61,7 +59,7 @@ public static void init() { TestHelper testHelper = TestHelper.getInstance(); transaction = testHelper.getTransaction(); capture = new StreamCapture( transaction ); - namespaceId = Helper.createNamespace( "testspace", NamespaceType.DOCUMENT ); + namespaceId = Helper.createNamespace(); Helper.createCollection(); } @@ -115,10 +113,9 @@ public void insertBooleanTest() { //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); BsonDocument result = Helper.filter( "{\"payload\":true}" ).get( 0 ); - List collection = Helper.scanCollection(); assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); - assertEquals( true, result.get( "payload" ).asBoolean().getValue() ); + assertTrue( result.get( "payload" ).asBoolean().getValue() ); assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); } @@ -133,7 +130,7 @@ public void insertJsonTest() { assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); assertEquals( "streamCaptureTest", result.get( "source" ).asString().getValue() ); assertEquals( "value1", result.get( "payload" ).asDocument().get( "key1" ).asString().getValue() ); - assertEquals( true, result.get( "payload" ).asDocument().get( "key2" ).asBoolean().getValue() ); + assertTrue( result.get( "payload" ).asDocument().get( "key2" ).asBoolean().getValue() ); assertEquals( 3, result.get( "payload" ).asDocument().get( "key3" ).asInt32().getValue() ); } @@ -146,7 +143,6 @@ public void insertArrayTest() { //StreamCapture capture = new StreamCapture( transaction ); capture.insert( storingmsg ); BsonDocument result = Helper.filter( "{\"payload\":[1, 2, 3]}" ).get( 0 ); - List collection = Helper.scanCollection(); assertEquals( "testTopic", result.get( "topic" ).asString().getValue() ); BsonArray expectedPayload = new BsonArray(); expectedPayload.add( 0, new BsonInt32( 1 ) ); @@ -178,9 +174,9 @@ public void isBooleanTest() { private static class Helper { - private static long createNamespace( String namespaceName, NamespaceType namespaceType ) { + private static long createNamespace() { Catalog catalog = Catalog.getInstance(); - long id = catalog.addNamespace( namespaceName, Catalog.defaultDatabaseId, Catalog.defaultUserId, namespaceType ); + long id = catalog.addNamespace( "testspace", Catalog.defaultDatabaseId, Catalog.defaultUserId, NamespaceType.DOCUMENT ); try { catalog.commit(); return id; @@ -208,23 +204,6 @@ private static void createCollection() { } - private static List scanCollection() { - String sqlCollectionName = "testSpace" + "." + "testCollection"; - Statement statement = transaction.createStatement(); - AlgBuilder builder = AlgBuilder.create( statement ); - - AlgNode algNode = builder.docScan( statement, sqlCollectionName ).build(); - - AlgRoot root = AlgRoot.of( algNode, Kind.SELECT ); - List> res = capture.executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); - List result = new ArrayList<>(); - for ( List objectsList : res ) { - result.add( objectsList.get( 0 ).toString() ); - } - return result; - } - - private static List filter( String query ) { Statement statement = transaction.createStatement(); QueryParameters parameters = new MqlQueryParameters( String.format( "db.%s.find(%s)", "testCollection", query ), "testSpace", NamespaceType.DOCUMENT ); From 968901174b316f8f7d6f695b06f1fb13783ba2ae Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Fri, 29 Sep 2023 19:47:33 +0200 Subject: [PATCH 109/114] reformatted files --- .../polypheny/db/stream/StreamMessage.java | 2 +- .../db/cypher/parser/CypherCharStream.java | 2 +- plugins/mqtt-stream/build.gradle | 5 +---- .../db/mqtt/FilteringMqttMessage.java | 2 +- .../org/polypheny/db/mqtt/MqttMessage.java | 2 +- .../polypheny/db/mqtt/MqttStreamPlugin.java | 21 ------------------- .../polypheny/db/mqtt/StoringMqttMessage.java | 9 +++----- .../polypheny/db/webui/crud/LanguageCrud.java | 2 +- 8 files changed, 9 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java index fc6fa0f6b2..2a39f09361 100644 --- a/core/src/main/java/org/polypheny/db/stream/StreamMessage.java +++ b/core/src/main/java/org/polypheny/db/stream/StreamMessage.java @@ -19,5 +19,5 @@ public interface StreamMessage { - T getMessage(); + T getData(); } diff --git a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/parser/CypherCharStream.java b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/parser/CypherCharStream.java index 0db4857162..6805c3b6a2 100644 --- a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/parser/CypherCharStream.java +++ b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/parser/CypherCharStream.java @@ -170,7 +170,7 @@ private char convertUnicode( char c ) { hexval( nextQueryChar() )); } catch ( final IOException e ) { throw new RuntimeException( e.getMessage() ); - //throw new RuntimeException( e.getMessage(), queryCursor, queryCursorLine, queryCursorColumn ); + //throw new RuntimeException( e.getData(), queryCursor, queryCursorLine, queryCursorColumn ); } } diff --git a/plugins/mqtt-stream/build.gradle b/plugins/mqtt-stream/build.gradle index cd3f63f88f..31e4f4450b 100644 --- a/plugins/mqtt-stream/build.gradle +++ b/plugins/mqtt-stream/build.gradle @@ -14,9 +14,6 @@ dependencies { // https://mvnrepository.com/artifact/com.hivemq/hivemq-mqtt-client implementation group: 'com.hivemq', name: 'hivemq-mqtt-client', version: hivemq_mqttclient_version - //TODO: remove - // https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on - implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.70' implementation group: "org.mongodb", name: "mongodb-driver-sync", version: mongodb_driver_sync_version // Apache 2.0 @@ -94,7 +91,7 @@ jar { } } -java{ +java { withJavadocJar(); withSourcesJar(); } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java index 900cc56de7..474f45d229 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/FilteringMqttMessage.java @@ -32,7 +32,7 @@ public FilteringMqttMessage( MqttMessage mqttMessage, String query ) { public String getMessage() { - return mqttMessage.getMessage(); + return mqttMessage.getData(); } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java index 9d06a5ddab..614d6f1aff 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttMessage.java @@ -33,7 +33,7 @@ public MqttMessage( String payload, String topic ) { } - public String getMessage() { + public String getData() { return this.payload; } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 84235516da..93f2de734b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -753,11 +753,9 @@ private class MonitoringPage { private final InformationGroup informationGroupInfo; private final InformationGroup informationGroupReceivedMessages; private final InformationGroup informationGroupPub; - private final InformationGroup informationGroupReconn; private final InformationTable topicsTable; private final InformationTable messageTable; private final InformationKeyValue brokerKv; - private final InformationAction reconnButton; private final InformationAction msgButton; @@ -797,23 +795,6 @@ public MonitoringPage() { } ).withParameters( "topic", "payload" ); im.registerInformation( msgButton ); - // Reconnection button - informationGroupReconn = new InformationGroup( informationPage, "Reconnect to broker" ).setOrder( 5 ); - im.addGroup( informationGroupReconn ); - reconnButton = new InformationAction( informationGroupReconn, "Reconnect", ( parameters ) -> { - String end = "Reconnecting to broker"; - if ( client.getState().toString().equals( "DISCONNECTED" ) ) { - run(); - update(); - } else { - client.toBlocking().disconnect(); - run(); - update(); - } - return end; - } ); - im.registerInformation( reconnButton ); - } @@ -852,11 +833,9 @@ public void remove() { im.removeInformation( brokerKv ); im.removeInformation( messageTable ); im.removeInformation( msgButton ); - im.removeInformation( reconnButton ); im.removeGroup( informationGroupTopics ); im.removeGroup( informationGroupInfo ); - im.removeGroup( informationGroupReconn ); im.removeGroup( informationGroupPub ); im.removeGroup( informationGroupReceivedMessages ); im.removePage( informationPage ); diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java index ff1fbc8398..6fe1b9a7c6 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java @@ -33,11 +33,8 @@ public class StoringMqttMessage { private final long databaseId; @Getter private final int userId; - /** - * if MqttStreamClient.collectionPerTopic = TRUE, then collectionName is name of the topic or (if the subscribed topic - * has wildcards) the wildcardTopic - * if MqttStreamClient.collectionPerTopic = FALSE, then collectionName is the name of the common collection - */ + + // The name of the entity where the message should be stored in. @Getter private final String entityName; @@ -54,7 +51,7 @@ public StoringMqttMessage( MqttMessage msg, String namespaceName, NamespaceType public String getMessage() { - return this.msg.getMessage(); + return this.msg.getData(); } diff --git a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java index fde6f84be0..2f210312eb 100644 --- a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java +++ b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java @@ -162,7 +162,7 @@ public static void printLog( Throwable t, QueryRequest request ) { public static void attachError( Transaction transaction, List results, String query, Throwable t ) { - //String msg = t.getMessage() == null ? "" : t.getMessage(); + //String msg = t.getData() == null ? "" : t.getData(); Result result = new Result( t ).setGeneratedQuery( query ).setXid( transaction.getXid().toString() ); if ( transaction.isActive() ) { From f14cb74b4694b82e588b01b424049ce1f3db133a Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sat, 30 Sep 2023 09:08:46 +0200 Subject: [PATCH 110/114] changed test method name and made few modifications --- .../db/mqtt/MqttClientBrokerTest.java | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java index 9ba681d913..5a08b45abc 100644 --- a/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java +++ b/plugins/mqtt-stream/src/test/java/org/polypheny/db/mqtt/MqttClientBrokerTest.java @@ -18,14 +18,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicLong; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -121,7 +118,7 @@ private void simulateIoTDevices() { @Test - public void simpleSubscribeUnsubscribeTest() { + public void subscribeUnsubscribeTest() { changedSettings.replace( "topics", "device1/sensor/battery" ); //All subscribed topics so far are unsubscribed client.updateSettings( changedSettings ); @@ -134,7 +131,7 @@ public void simpleSubscribeUnsubscribeTest() { @Test - public void subscribeWithWildcardHashtagTest() { + public void topicMapUpdatedCorrectlyWithWildcardHashtagTest() { changedSettings.replace( "topics", "#" ); client.updateSettings( changedSettings ); simulateIoTDevices(); @@ -143,50 +140,38 @@ public void subscribeWithWildcardHashtagTest() { @Test - public void subscribeWithWildcardHashtagAtEndTest() { + public void topicMapUpdatedCorrectlyWildcardHashtagAtEndTest() { changedSettings.replace( "topics", "device1/#" ); client.updateSettings( changedSettings ); simulateIoTDevices(); assertEquals( 4, client.getTopicsMap().get( "device1/#" ).intValue() ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/online", "true" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements/unit", "C" } ) ); } @Test - public void subscribeWithWildcardPlusAtEndTest() { + public void topicMapUpdatedCorrectlyWithWildcardPlusAtEndTest() { changedSettings.replace( "topics", "device1/sensor/+" ); client.updateSettings( changedSettings ); simulateIoTDevices(); assertEquals( 3, client.getTopicsMap().get( "device1/sensor/+" ).intValue() ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/battery", "86" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements", "[28,76,55 ]" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/sensor/measurements/unit", "C" } ) ); } @Test - public void subscribeWithWildcardPlusInMiddleTest() { + public void topicMapUpdatedCorrectlyWithWildcardPlusInMiddleTest() { changedSettings.replace( "topics", "device2/+/info" ); client.updateSettings( changedSettings ); simulateIoTDevices(); assertEquals( 2, client.getTopicsMap().get( "device2/+/info" ).intValue() ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device2/location/info", "Basel" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device2/sensor/info", "{\"wifi\":\"networkName\", \"mqtt\":{\"brokerIp\":\"127.0.0.1\", \"port\":1883}, \"deviceName\":\"device2\"}" } ) ); - } @Test - public void subscribeWithWildcardPlusAtBeginningTest() { + public void topicMapUpdatedCorrectlyWithWildcardPlusAtBeginningTest() { changedSettings.replace( "topics", "+/online" ); client.updateSettings( changedSettings ); simulateIoTDevices(); assertEquals( 2, client.getTopicsMap().get( "+/online" ).intValue() ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device1/online", "true" } ) ); - assertTrue( client.getMessageQueue().contains( new String[]{ "device2/online", "true" } ) ); } @@ -219,7 +204,6 @@ public void reloadSettingsCatchAllEntityToFalseTest() { @Test public void getWildcardTopicWithHashtagTest() { changedSettings.replace( "topics", "device1/sensor/#"); - changedSettings.replace( "filterQuery", "device1/sensor/#:{}" ); client.updateSettings( changedSettings ); String resultWildcardTopic1 = client.getWildcardTopic( "device1/sensor/measurements/unit" ); assertEquals( "device1/sensor/#", resultWildcardTopic1 ); @@ -229,7 +213,6 @@ public void getWildcardTopicWithHashtagTest() { @Test public void getWildcardTopicWithPlusTest() { changedSettings.replace( "topics", "+/online"); - changedSettings.replace( "filterQuery", "+/online:{\"$$ROOT\":false}" ); client.updateSettings( changedSettings ); String resultWildcardTopic = client.getWildcardTopic( "device1/online" ); assertEquals( "+/online", resultWildcardTopic ); From 7e524ad484e12c8e4a806b7ce49177c67204d119 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 8 Oct 2023 10:44:59 +0200 Subject: [PATCH 111/114] fixed mistake in UI --- .../src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 93f2de734b..5a1a1b396b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -805,8 +805,7 @@ public void update() { topicsTable.addRow( "No topic subscriptions" ); } else { for ( Entry t : topicsMap.entrySet() ) { - String filterQuery = filterMap.get( t.getKey() ); - topicsTable.addRow( t.getKey(), t.getValue(), filterQuery == null ? "" : filterQuery ); + topicsTable.addRow( t.getKey(), t.getValue() ); } } From 002b814620e7b5bcaba5630a8b9d18d0029b8785 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Sun, 8 Oct 2023 10:45:16 +0200 Subject: [PATCH 112/114] removed forgotten logger --- .../src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java index ba12ca1d96..131846b6a8 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamProcessor.java @@ -53,7 +53,6 @@ public MqttStreamProcessor( FilteringMqttMessage filteringMqttMessage, Statement public boolean applyFilter() { AlgRoot root = processMqlQuery(); List> res = executeAndTransformPolyAlg( root, statement ); - log.info( res.toString() ); return res.size() != 0; } From 5c3723e3ed90b17b3dd8853e651882efae0d9da4 Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Mon, 16 Oct 2023 18:32:02 +0200 Subject: [PATCH 113/114] cleaned up code --- .../polypheny/db/mqtt/MqttStreamPlugin.java | 23 ++----- .../polypheny/db/mqtt/StoringMqttMessage.java | 2 +- .../org/polypheny/db/mqtt/StreamCapture.java | 63 +++++++++---------- 3 files changed, 36 insertions(+), 52 deletions(-) diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java index 5a1a1b396b..627b74312b 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/MqttStreamPlugin.java @@ -445,22 +445,6 @@ private void subscribe( String topic ) { .send() .whenComplete( ( subAck, throwable ) -> { if ( throwable != null ) { - //TODO: change settings correctly: Test this - List topicsList = toList( this.getCurrentSettings().get( "topics" ) ); - StringBuilder stringBuilder = new StringBuilder(); - for ( String t : topicsList ) { - if ( !t.equals( topic ) ) { - stringBuilder.append( t ).append( "," ); - } - } - String topicsString = stringBuilder.toString(); - if ( topicsString != null && !topicsString.isBlank() ) { - topicsString = topicsString.substring( 0, topicsString.lastIndexOf( ',' ) ); - } - synchronized ( settingsLock ) { - this.settings.put( "topics", topicsString ); - } - log.info( "not successful: {}", topic ); throw new RuntimeException( String.format( "Subscription was not successful for topic \"%s\" . Please try again.", topic ), throwable ); } else { this.topicsMap.put( topic, new AtomicLong( 0 ) ); @@ -598,12 +582,12 @@ protected static String extractPayload( Mqtt5Publish subMsg ) { protected String getWildcardTopic( String topic ) { for ( String t : topicsMap.keySet() ) { - //multilevel wildcard + //check for multilevel wildcard if ( t.contains( "#" ) && topic.startsWith( t.substring( 0, t.indexOf( "#" ) ) ) ) { return t; } - // single level wildcard + // check for single level wildcard if ( t.contains( "+" ) && topic.startsWith( t.substring( 0, t.indexOf( "+" ) ) ) && topic.endsWith( t.substring( t.indexOf( "+" ) + 1 ) ) ) { return t; } @@ -677,7 +661,7 @@ private void createEntity( String entityName ) { DdlManager.getInstance().createCollection( namespaceID, collectionName, - true, //only creates collection if it does not already exist. + true, dataStores.size() == 0 ? null : dataStores, PlacementType.MANUAL, statement ); @@ -712,6 +696,7 @@ private void createAllEntities() { } } else { // handle other namespace types + throw new RuntimeException("Other namespace types are not implemented yet"); } } diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java index 6fe1b9a7c6..8fb45c8861 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StoringMqttMessage.java @@ -19,7 +19,7 @@ import lombok.Getter; import org.polypheny.db.catalog.Catalog.NamespaceType; -//TODO: javadoc + public class StoringMqttMessage { private final MqttMessage msg; diff --git a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java index 9c9dd5a52d..459bc04e34 100644 --- a/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java +++ b/plugins/mqtt-stream/src/main/java/org/polypheny/db/mqtt/StreamCapture.java @@ -41,7 +41,6 @@ public class StreamCapture { Transaction transaction; - PolyphenyHomeDirManager homeDirManager; StoringMqttMessage storingMqttMessage; @@ -60,36 +59,8 @@ private void insertMessage() { if ( this.storingMqttMessage.getNamespaceType() == NamespaceType.DOCUMENT ) { String sqlCollectionName = this.storingMqttMessage.getNamespaceName() + "." + this.storingMqttMessage.getEntityName(); Statement statement = transaction.createStatement(); - - // Builder which allows to construct the algebra tree which is equivalent to query and is executed - AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); - - BsonDocument document = new BsonDocument(); - document.put( "source", new BsonString( this.storingMqttMessage.getUniqueNameOfInterface() ) ); - document.put( "topic", new BsonString( this.storingMqttMessage.getTopic() ) ); - String msg = this.storingMqttMessage.getMessage(); - BsonValue value; - if ( msg.contains( "{" ) && msg.contains( "}" ) ) { - value = BsonDocument.parse( msg ); - } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { - BsonArray bsonArray = new BsonArray(); - msg = msg.replace( "[", "" ).replace( "]", "" ); - String[] msglist = msg.split( "," ); - for ( String stringValue : msglist ) { - stringValue = stringValue.trim(); - bsonArray.add( getBsonValue( stringValue ) ); - } - value = bsonArray; - } else { - // msg is a single value - value = getBsonValue( msg ); - } - document.put( "payload", value ); - - AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); - + AlgNode algNode = createDocument( statement, sqlCollectionName ); AlgRoot root = AlgRoot.of( algNode, Kind.INSERT ); - // for inserts and all DML queries only a number is returned List> res = executeAndTransformPolyAlg( root, statement, statement.getPrepareContext() ); try { transaction.commit(); @@ -100,11 +71,39 @@ private void insertMessage() { } + private AlgNode createDocument( Statement statement, String sqlCollectionName) { + AlgBuilder builder = AlgBuilder.createDocumentBuilder( statement ); + + BsonDocument document = new BsonDocument(); + document.put( "source", new BsonString( this.storingMqttMessage.getUniqueNameOfInterface() ) ); + document.put( "topic", new BsonString( this.storingMqttMessage.getTopic() ) ); + String msg = this.storingMqttMessage.getMessage(); + BsonValue value; + if ( msg.contains( "{" ) && msg.contains( "}" ) ) { + value = BsonDocument.parse( msg ); + } else if ( msg.contains( "[" ) && msg.contains( "]" ) ) { + BsonArray bsonArray = new BsonArray(); + msg = msg.replace( "[", "" ).replace( "]", "" ); + String[] msglist = msg.split( "," ); + for ( String stringValue : msglist ) { + stringValue = stringValue.trim(); + bsonArray.add( getBsonValue( stringValue ) ); + } + value = bsonArray; + } else { + // msg is a single value + value = getBsonValue( msg ); + } + document.put( "payload", value ); + + AlgNode algNode = builder.docInsert( statement, sqlCollectionName, document ).build(); + return algNode; + } + + /** * turns one single value into the corresponding BsonValue - * * @param value value that has to be casted as String - * @return */ protected BsonValue getBsonValue( String value ) { if ( isInteger( value ) ) { From 15ee90fef45e4e661cd125da1305b59890cef5cd Mon Sep 17 00:00:00 2001 From: Neha Selvan Date: Tue, 17 Oct 2023 19:05:46 +0200 Subject: [PATCH 114/114] removed code related to stream entity --- .../org/polypheny/db/catalog/Catalog.java | 4 +- .../java/org/polypheny/db/ddl/DdlManager.java | 2 - .../db/schema/LogicalStreamCollection.java | 49 ------------------ .../org/polypheny/db/ddl/DdlManagerImpl.java | 50 ------------------- .../db/schema/PolySchemaBuilder.java | 8 --- 5 files changed, 2 insertions(+), 111 deletions(-) delete mode 100644 core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java diff --git a/core/src/main/java/org/polypheny/db/catalog/Catalog.java b/core/src/main/java/org/polypheny/db/catalog/Catalog.java index 44f984eec4..505d11fcf4 100644 --- a/core/src/main/java/org/polypheny/db/catalog/Catalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/Catalog.java @@ -1933,8 +1933,8 @@ public enum EntityType { ENTITY( 1 ), SOURCE( 2 ), VIEW( 3 ), - MATERIALIZED_VIEW( 4 ), - STREAM(5); + MATERIALIZED_VIEW( 4 ); + //STREAM(5); private final int id; diff --git a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java index 4fd45b9f20..0e6741415f 100644 --- a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java +++ b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java @@ -476,8 +476,6 @@ public static DdlManager getInstance() { public abstract void createCollection( long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException; - public abstract void createStreamCollection(long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException; - public abstract void addCollectionPlacement( long namespaceId, String name, List stores, Statement statement ); /** diff --git a/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java b/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java deleted file mode 100644 index 8cb051faf7..0000000000 --- a/core/src/main/java/org/polypheny/db/schema/LogicalStreamCollection.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019-2022 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.schema; - -import java.util.List; -import org.polypheny.db.algebra.AlgNode; -import org.polypheny.db.algebra.core.Modify; -import org.polypheny.db.algebra.core.Modify.Operation; -import org.polypheny.db.algebra.type.AlgProtoDataType; -import org.polypheny.db.catalog.Catalog.NamespaceType; -import org.polypheny.db.plan.AlgOptCluster; -import org.polypheny.db.plan.AlgOptTable; -import org.polypheny.db.prepare.Prepare.CatalogReader; -import org.polypheny.db.rex.RexNode; - - -public class LogicalStreamCollection extends LogicalTable { - - protected LogicalStreamCollection( long tableId, String logicalSchemaName, String logicalTableName, AlgProtoDataType protoRowType ) { - super( tableId, logicalSchemaName, logicalTableName, List.of( 0L ), List.of( "d" ), protoRowType, NamespaceType.DOCUMENT ); - } - - @Override - public Modify toModificationAlg( - AlgOptCluster cluster, - AlgOptTable table, - CatalogReader catalogReader, - AlgNode input, - Operation operation, - List updateColumnList, - List sourceExpressionList, - boolean flattened ) { - throw new RuntimeException( "Collection of type STREAM can not be modified." ); - } -} diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 094facf45e..338d484a69 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -2293,56 +2293,6 @@ public void createCollection( long schemaId, String name, boolean ifNotExists, L } - @Override - public void createStreamCollection( long schemaId, String name, boolean ifNotExists, List stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException { - name = adjustNameIfNeeded( name, schemaId ); - - if ( assertEntityExists( schemaId, name, ifNotExists ) ) { - return; - } - - if ( stores == null ) { - // Ask router on which store(s) the table should be placed - stores = RoutingManager.getInstance().getCreatePlacementStrategy().getDataStoresForNewTable(); - } - - long collectionId; - long partitionId; - try { - collectionId = catalog.addCollectionLogistics( schemaId, name, stores, false ); - partitionId = catalog.getPartitionGroups( collectionId ).get( 0 ).id; - } catch ( GenericCatalogException e ) { - throw new RuntimeException( e ); - } - - catalog.addCollection( - collectionId, - name, - schemaId, - statement.getPrepareContext().getCurrentUserId(), - EntityType.STREAM, - true ); - - // Initially create DataPlacement containers on every store the table should be placed. - CatalogCollection catalogCollection = catalog.getCollection( collectionId ); - - // Trigger rebuild of schema; triggers schema creation on adapters - PolySchemaBuilder.getInstance().getCurrent(); - - for ( DataStore store : stores ) { - catalog.addCollectionPlacement( - store.getAdapterId(), - catalogCollection.id, - PlacementType.AUTOMATIC ); - - afterDocumentLogistics( store, collectionId ); - - store.createCollection( statement.getPrepareContext(), catalogCollection, store.getAdapterId() ); - } - - } - - private boolean assertEntityExists( long namespaceId, String name, boolean ifNotExists ) throws EntityAlreadyExistsException { // Check if there is already an entity with this name if ( catalog.checkIfExistsEntity( namespaceId, name ) ) { diff --git a/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java b/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java index 825a417f2b..b677adc0fa 100644 --- a/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java +++ b/dbms/src/main/java/org/polypheny/db/schema/PolySchemaBuilder.java @@ -211,14 +211,6 @@ private void buildDocumentLogical( AbstractPolyphenyDbSchema polyphenyDbSchema, catalogEntity.getNamespaceName(), catalogEntity.name, AlgDataTypeImpl.proto( fieldInfo.build() ) ); - - } else if ( catalogEntity.entityType == EntityType.STREAM ) { - entity = new LogicalStreamCollection( - catalogEntity.id, - catalogEntity.getNamespaceName(), - catalogEntity.name, - AlgDataTypeImpl.proto( fieldInfo.build() ) ); - } else { throw new RuntimeException( "Unhandled table type: " + catalogEntity.entityType.name() ); }