From 6544341089362d1f568208d03f3955b7a4b62bbd Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 10:21:46 +0200 Subject: [PATCH 01/15] project structure --- .gitignore | 9 + eclipse-java-formatter.xml | 450 +++++++++++++++++++++++++++++ license-header | 45 +++ pom.xml | 575 +++++++++++++++++++++++++++++++++++++ 4 files changed, 1079 insertions(+) create mode 100644 .gitignore create mode 100644 eclipse-java-formatter.xml create mode 100644 license-header create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4774b04 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/.settings/* +/.vscode/* +/.flattened-pom.xml +/target/* +/.classpath +/.factorypath +/.project +/.idea/ +/*.iml diff --git a/eclipse-java-formatter.xml b/eclipse-java-formatter.xml new file mode 100644 index 0000000..53b9f2a --- /dev/null +++ b/eclipse-java-formatter.xml @@ -0,0 +1,450 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/license-header b/license-header new file mode 100644 index 0000000..51ba6ec --- /dev/null +++ b/license-header @@ -0,0 +1,45 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..903ca79 --- /dev/null +++ b/pom.xml @@ -0,0 +1,575 @@ + + + 4.0.0 + com.teragrep + pth_16 + ${revision}${sha1}${changelist} + jar + pth_16 + Teragrep DPL Command-Line Executor + https://teragrep.com + + + GNU Affero General Public License v3.0 + https://www.gnu.org/licenses/agpl-3.0.txt + + + + + Mikko Kortelainen + 1@teragrep.com + Teragrep + https://teragrep.com + + + + scm:git:https://github.com/teragrep/pth_16.git + scm:git:git@github.com:teragrep/pth_16.git + https://github.com/teragrep/pth_16/tree/master + + + -SNAPSHOT + 1.8 + 2.3.0 + 1.8 + 1.8 + 4.2.0 + UTF-8 + 0.0.2 + + + + + org.junit.jupiter + junit-jupiter-engine + 5.4.0-RC1 + test + + + org.junit.platform + junit-platform-launcher + 1.4.0-RC1 + test + + + org.junit.jupiter + junit-jupiter-api + 5.4.0-RC1 + test + + + nl.jqno.equalsverifier + equalsverifier + 3.16.1 + test + + + + ${project.basedir}/target + pth_16 + + + org.apache.maven.plugins + maven-source-plugin + 3.3.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.6.3 + + + attach-javadocs + + jar + + + 8 + false + none + + + startuml + + startuml + + + enduml + + enduml + + + class + + class + + + brief + + brief + + + responsibilities + + responsibilities + + + collaborators + + collaborators + + + + + + + + org.apache.rat + apache-rat-plugin + 0.15 + false + + false + + + Also allow the license url to be https. + + GNU Affero General Public License + + + + true + false + + + .git/** + .gitattributes + .gitignore + .gitmodules + + .github/workflows/*.yml + .github/workflows/*.yaml + .github/ISSUE_TEMPLATE/* + .github/pull_request_template.md + + README.adoc + README.md + + Doxyfile + + + + + + check + + test + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.30.0 + + + + ${project.basedir}/eclipse-java-formatter.xml + 4.10.0 + + + + ${project.basedir}/license-header + + + + + + UTF-8 + \n + true + false + 2 + recommended_2008_06 + true + true + true + + + + + + .gitattributes + .gitignore + + + + + true + 4 + + + + + + + + check + + compile + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + bom + + + + flatten.clean + + clean + + clean + + + flatten + + flatten + + process-resources + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + -Xlint:all + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + all + true + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.5.0 + + + + scan-errors + + check + + process-classes + + error + true + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + scan-warnings + + check + + process-classes + + warning + true + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + publish-maven-central + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.5.0 + true + + central-sonatype-org + true + true + published + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + + sign + + verify + + + --pinentry-mode + loopback + + + + + + + + + + publish-github-packages + + + github + GitHub Packages + https://maven.pkg.github.com/teragrep/pth_16 + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + + sign + + verify + + + --pinentry-mode + loopback + + + + + + + + + + From 3339ee47529355227148d0e782634471e4279cb9 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 10:46:54 +0200 Subject: [PATCH 02/15] initial version --- pom.xml | 11 ++ src/main/java/com/teragrep/pth_16/Entry.java | 119 +++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 src/main/java/com/teragrep/pth_16/Entry.java diff --git a/pom.xml b/pom.xml index 903ca79..c61b65e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,6 +39,17 @@ + + com.teragrep + pth_15 + 2.0.0 + + + org.apache.spark + spark-sql_2.12 + 3.5.7 + provided + org.junit.jupiter junit-jupiter-engine diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java new file mode 100644 index 0000000..51248a0 --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -0,0 +1,119 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.teragrep.pth_15.DPLExecutor; +import com.teragrep.pth_15.DPLExecutorFactory; +import com.teragrep.pth_15.DPLExecutorResult; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; + +import java.lang.reflect.InvocationTargetException; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; + +public class Entry { + + public static void main(String[] args) { + Config config = ConfigFactory.parseProperties(new Properties()); + String applicationName = "com.teragrep.pth_16.Entry"; + String lines = "| makeresults count=1"; + String queryId = UUID.randomUUID().toString(); + String noteId = ""; + String paragraphId = ""; + + BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { + rowDataset.show(false); + }; + + final DPLExecutor dplExecutor; + try { + dplExecutor = new DPLExecutorFactory("com.teragrep.pth_10.executor.DPLExecutorImpl", config).create(); + } + catch ( + ClassNotFoundException | NoSuchMethodException | InvocationTargetException | InstantiationException + | IllegalAccessException e + ) { + throw new RuntimeException("Error initializing DPLExecutor implementation", e); + } + + SparkSession sparkSession = SparkSession.builder().appName(applicationName).getOrCreate(); + + try { + final DPLExecutorResult executorResult = dplExecutor + .interpret(batchHandler, sparkSession, queryId, noteId, paragraphId, lines); + + /* + final InterpreterResult.Code code; + if (executorResult.code().equals(DPLExecutorResult.Code.SUCCESS)) { + code = Code.SUCCESS; + } else if (executorResult.code().equals(DPLExecutorResult.Code.INCOMPLETE)) { + code = DPLExecutorResult.Code.INCOMPLETE; + } else if (executorResult.code().equals(DPLExecutorResult.Code.KEEP_PREVIOUS_RESULT)) { + code = Code.KEEP_PREVIOUS_RESULT; + } else { + code = Code.ERROR; + } + */ + + System.out + .println( + "executorResult code <" + executorResult.code() + "> message <" + executorResult.message() + + ">" + ); + + } + catch (TimeoutException e) { + throw new RuntimeException(e); + } + } + +} From 8f1673ad094e42b9ea2a92d168b18e2d8c2e387f Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 10:51:24 +0200 Subject: [PATCH 03/15] initial version --- pom.xml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c61b65e..537d3e8 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 1.8 4.2.0 UTF-8 - 0.0.2 + 0.0.1 @@ -79,6 +79,30 @@ ${project.basedir}/target pth_16 + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + jar-with-dependencies + + + + com.teragrep.pth_16.Entry + + + + + + make-assembly + + single + + package + + + org.apache.maven.plugins maven-source-plugin From 667365028630d2a6ed13b4ed8ab78f3780458fd4 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 12:51:55 +0200 Subject: [PATCH 04/15] ability to use zep_01 configuration file --- pom.xml | 11 ++ src/main/java/com/teragrep/pth_16/Entry.java | 13 +- .../pth_16/InterpreterSettingsConfig.java | 174 ++++++++++++++++++ .../pth_16/InterpreterSettingsConfigTest.java | 73 ++++++++ src/test/resources/interpreter.json | 85 +++++++++ 5 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java create mode 100644 src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java create mode 100644 src/test/resources/interpreter.json diff --git a/pom.xml b/pom.xml index 537d3e8..0a53155 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,17 @@ 3.5.7 provided + + jakarta.json + jakarta.json-api + 2.1.3 + + + org.eclipse.parsson + parsson + 1.1.5 + runtime + org.junit.jupiter junit-jupiter-engine diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index 51248a0..cfe1280 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -49,21 +49,26 @@ import com.teragrep.pth_15.DPLExecutorFactory; import com.teragrep.pth_15.DPLExecutorResult; import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import java.io.File; +import java.io.FileNotFoundException; import java.lang.reflect.InvocationTargetException; -import java.util.Properties; import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; public class Entry { - public static void main(String[] args) { - Config config = ConfigFactory.parseProperties(new Properties()); + public static void main(String[] args) throws FileNotFoundException { + File configFile = new File("interpreter.json"); + + InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); + + Config config = interpreterSettingsConfig.getConfig(); + String applicationName = "com.teragrep.pth_16.Entry"; String lines = "| makeresults count=1"; String queryId = UUID.randomUUID().toString(); diff --git a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java new file mode 100644 index 0000000..9b38df4 --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java @@ -0,0 +1,174 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import java.io.File; + +import com.typesafe.config.ConfigValueFactory; +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonReader; +import jakarta.json.JsonValue; + +import java.io.FileReader; +import java.io.FileNotFoundException; +import java.util.Map; + +public class InterpreterSettingsConfig { + + private final File configFile; + + public InterpreterSettingsConfig(File configFile) { + this.configFile = configFile; + } + + public Config getConfig() throws FileNotFoundException { + + Config config = ConfigFactory.empty(); + + try (JsonReader reader = Json.createReader(new FileReader(configFile))) { + JsonObject jsonObject = reader.readObject(); + + if (!jsonObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException("config does not contain a JSON object"); + } + + String interpreterSettingsKey = "interpreterSettings"; + if (!jsonObject.containsKey(interpreterSettingsKey)) { + throw new IllegalArgumentException( + "config does not contain a JSON object with key <" + interpreterSettingsKey + ">" + ); + } + + JsonObject interpreterSettings = jsonObject.getJsonObject(interpreterSettingsKey); + + String sparkKey = "spark"; + if (!interpreterSettings.containsKey(sparkKey)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + ">" + ); + } + + JsonObject sparkObject = interpreterSettings.getJsonObject(sparkKey); + + if (!sparkObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + + "> as a JSON object" + ); + } + + String propertieskey = "properties"; + if (!sparkObject.containsKey(propertieskey)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + + "> with key <" + propertieskey + ">" + ); + } + + JsonObject propertiesObject = sparkObject.getJsonObject(propertieskey); + + if (!propertiesObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + + "> with key <" + propertieskey + "> as a JSON object" + ); + } + + for (Map.Entry entry : propertiesObject.entrySet()) { + if (entry.getKey().startsWith("dpl.") || entry.getKey().startsWith("fs.s3a.")) { + String key = entry.getKey(); + + JsonValue entryValue = entry.getValue(); + + if (!entryValue.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException("json key <[" + key + "]> does not refer to a json object"); + } + + JsonObject entryValueObject = entryValue.asJsonObject(); + + String valueKey = "value"; + if (!entryValueObject.containsKey(valueKey)) { + throw new IllegalArgumentException( + "json key <[" + key + "]> does not refer to a json object with a key named <" + valueKey + + ">" + ); + } + JsonValue jsonValue = entryValueObject.get(valueKey); + + switch (jsonValue.getValueType()) { + case STRING: + config = config + .withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getString("value"))); + break; + case NULL: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(null)); + break; + case TRUE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(true)); + break; + case FALSE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(false)); + break; + case NUMBER: + config = config + .withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getInt("value"))); + break; + default: + throw new IllegalArgumentException( + "json key <[" + key + "]> has object with key <" + valueKey + + "> that has unsupported value type" + jsonValue.getValueType() + ); + } + } + } + } + return config; + } +} diff --git a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java new file mode 100644 index 0000000..f79e425 --- /dev/null +++ b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java @@ -0,0 +1,73 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.File; + +public class InterpreterSettingsConfigTest { + + @Test + public void loadNFilterZep01InterpreterSettings() { + InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig( + new File("src/test/resources/interpreter.json") + ); + + Assertions.assertDoesNotThrow(() -> { + Config config = interpreterSettingsConfig.getConfig(); + + Assertions.assertFalse(config.getBoolean("dpl.pth_06.sql.log.enabled")); + + Assertions.assertEquals("putUserHere", config.getString("fs.s3a.access.key")); + + Assertions.assertFalse(config.hasPath("spark.app.name")); + }); + } + +} diff --git a/src/test/resources/interpreter.json b/src/test/resources/interpreter.json new file mode 100644 index 0000000..0d6a248 --- /dev/null +++ b/src/test/resources/interpreter.json @@ -0,0 +1,85 @@ +{ + "interpreterSettings": { + "spark": { + "id": "spark", + "name": "spark", + "group": "spark", + "properties": { + "spark.app.name": { + "name": "spark.app.name", + "value": "pth_16", + "type": "string", + "description": "The name of spark application." + }, + "fs.s3a.access.key": { + "name": "fs.s3a.access.key", + "value": "putUserHere", + "type": "string", + "description": "S3 access key" + }, + "dpl.pth_06.sql.log.enabled": { + "name": "dpl.pth_06.sql.log.enabled", + "value": false, + "type": "checkbox" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "com.teragrep.zep_01.spark.SparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sql", + "class": "com.teragrep.zep_01.spark.SparkSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "pyspark", + "class": "com.teragrep.zep_01.spark.PySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "dpl", + "class": "com.teragrep.pth_07.DPLInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "dpl", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + } + ], + "option": { + "remote": true, + "port": -1, + "perNote": "isolated", + "perUser": "isolated", + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": true + } + } + } +} From b0d0aca0ba28529ba9d2e68add8a8be7c3b74950 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:12:51 +0200 Subject: [PATCH 05/15] custom query support --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.adoc | 57 ++++++++++++++++---- src/main/java/com/teragrep/pth_16/Entry.java | 14 +++-- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 5e14671..db35714 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -4,7 +4,7 @@ contact_links: url: https://github.com/teragrep/doc_01/issues/new?template=doc-issue-report.md about: Problems with Teragrep documentation - name: Ask a question or get support - url: https://github.com/teragrep/repo-template/discussions + url: https://github.com/teragrep/pth_16/discussions about: Ask a question or request support - name: Report vulnerability url: https://github.com/teragrep/teragrep/security/advisories/new diff --git a/README.adoc b/README.adoc index 0f563cc..10bf029 100644 --- a/README.adoc +++ b/README.adoc @@ -1,13 +1,7 @@ -// Before publishing your new repository: -// 1. Write the readme file -// 2. Update the issues link in Contributing section in the readme file -// 3. Update the discussion link in config.yml file in .github/ISSUE_TEMPLATE directory -= repo-template += Teragrep DPL Command-Line Executor (pth_16) -// Add a short description of your project. Tell what your project does and what it's used for. - -This is a template repository for Teragrep organization. +Teragrep DPL Command-Line Executor allows running Teragrep queries from command-line. == Features @@ -19,17 +13,58 @@ See the official documentation on https://docs.teragrep.com[docs.teragrep.com]. == Limitations -// If your project has limitations, please list them. Otherwise remove this section. +Uses spark-submit. == How to [compile/use/implement] -// add instructions how people can start to use your project +=== Usage + +Example without arguments + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar +---- + +Results into + +[source,text] +---- ++-------------------+--------------------+ +|_time |_raw | ++-------------------+--------------------+ +|2026-01-27 13:00:00|Welcome to Teragrep®| ++-------------------+--------------------+ +---- + +Custom query + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar \ + '| makeresults count=2 | eval _raw="this is a custom query"' + +---- == Contributing // Change the repository name in the issues link to match with your project's name -You can involve yourself with our project by https://github.com/teragrep/repo-template/issues/new/choose[opening an issue] or submitting a pull request. +You can involve yourself with our project by https://github.com/teragrep/pth_16/issues/new/choose[opening an issue] or submitting a pull request. Contribution requirements: diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index cfe1280..e4fedb4 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -70,10 +70,18 @@ public static void main(String[] args) throws FileNotFoundException { Config config = interpreterSettingsConfig.getConfig(); String applicationName = "com.teragrep.pth_16.Entry"; - String lines = "| makeresults count=1"; + + final String lines; + if (args.length > 0) { + lines = args[0]; + } + else { + lines = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; + } + String queryId = UUID.randomUUID().toString(); - String noteId = ""; - String paragraphId = ""; + String noteId = "pth_16-notebook-" + UUID.randomUUID(); + String paragraphId = "pth_16-paragraph-" + UUID.randomUUID(); BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { rowDataset.show(false); From 07cd74cd671c1fcb4b15dcf00118ae2b6aea343a Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:16:55 +0200 Subject: [PATCH 06/15] improve readme --- README.adoc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 10bf029..843dd83 100644 --- a/README.adoc +++ b/README.adoc @@ -56,10 +56,21 @@ JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ --total-executor-cores 8 \ --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ --files interpreter.json pth_16-jar-with-dependencies.jar \ - '| makeresults count=2 | eval _raw="this is a custom query"' + '| makeresults count=1 | eval _raw="this is a custom query"' ---- +Results into + +[source,text] +---- ++-------------------+----------------------+ +|_time |_raw | ++-------------------+----------------------+ +|2026-01-27 13:02:07|this is a custom query| ++-------------------+----------------------+ +---- + == Contributing // Change the repository name in the issues link to match with your project's name From 2de5cb8442e6097e9a83e1ba43fe09006660a8b6 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:18:47 +0200 Subject: [PATCH 07/15] improve readme --- README.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 843dd83..7cbbcd5 100644 --- a/README.adoc +++ b/README.adoc @@ -1,11 +1,11 @@ = Teragrep DPL Command-Line Executor (pth_16) -Teragrep DPL Command-Line Executor allows running Teragrep queries from command-line. +Teragrep DPL Command-Line Executor is an utility to run DPL queries from command-line. == Features -// List your project's features +- Allows running Teragrep queries from command-line. == Documentation From 25bd260a591d8d06de69cd9bf113fe57d4e517b7 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:39:13 +0200 Subject: [PATCH 08/15] change output to JSON --- src/main/java/com/teragrep/pth_16/Entry.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index e4fedb4..bf85381 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -56,8 +56,11 @@ import java.io.File; import java.io.FileNotFoundException; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; public class Entry { @@ -83,8 +86,11 @@ public static void main(String[] args) throws FileNotFoundException { String noteId = "pth_16-notebook-" + UUID.randomUUID(); String paragraphId = "pth_16-paragraph-" + UUID.randomUUID(); + final AtomicReference> rows = new AtomicReference<>(new ArrayList<>()); + BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { - rowDataset.show(false); + rows.set(rowDataset.toJSON().collectAsList()); + //rowDataset.show(false); }; final DPLExecutor dplExecutor; @@ -123,6 +129,9 @@ public static void main(String[] args) throws FileNotFoundException { + ">" ); + for (String string : rows.get()) { + System.out.println(string); + } } catch (TimeoutException e) { throw new RuntimeException(e); From 9769b94b6fd354b3de56fdba25513586eab11fd5 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:41:53 +0200 Subject: [PATCH 09/15] change output to JSON in README.adoc as well --- README.adoc | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.adoc b/README.adoc index 7cbbcd5..87f3079 100644 --- a/README.adoc +++ b/README.adoc @@ -35,13 +35,9 @@ JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ Results into -[source,text] +[source,json] ---- -+-------------------+--------------------+ -|_time |_raw | -+-------------------+--------------------+ -|2026-01-27 13:00:00|Welcome to Teragrep®| -+-------------------+--------------------+ +{"_time":"2026-01-27T13:37:00.000+02:00","_raw":"Welcome to Teragrep®"} ---- Custom query @@ -62,13 +58,9 @@ JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ Results into -[source,text] +[source,json] ---- -+-------------------+----------------------+ -|_time |_raw | -+-------------------+----------------------+ -|2026-01-27 13:02:07|this is a custom query| -+-------------------+----------------------+ +{"_time":"2026-01-27T13:37:54.000+02:00","_raw":"this is a custom query"} ---- == Contributing From c6311d23aa03c5088818316e6acf5628b807dd51 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 13:57:22 +0200 Subject: [PATCH 10/15] add pipe output to output.json example --- README.adoc | 25 ++++++++++++++++++++ src/main/java/com/teragrep/pth_16/Entry.java | 25 +++++--------------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/README.adoc b/README.adoc index 87f3079..a6ea6ab 100644 --- a/README.adoc +++ b/README.adoc @@ -63,6 +63,31 @@ Results into {"_time":"2026-01-27T13:37:54.000+02:00","_raw":"this is a custom query"} ---- +Results can be piped into a file + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar \ + '| makeresults count=1 | eval _raw="this is a custom query"' | tee output.json +---- + +Results into + +[source,json] +---- +{ + "_time": "2026-01-27T13:54:24.000+02:00", + "_raw": "this is a custom query" +} +---- + == Contributing // Change the repository name in the issues link to match with your project's name diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index bf85381..5e1c9c9 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -52,6 +52,8 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; @@ -66,6 +68,9 @@ public class Entry { public static void main(String[] args) throws FileNotFoundException { + + Logger LOGGER = LoggerFactory.getLogger(Entry.class); + File configFile = new File("interpreter.json"); InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); @@ -90,7 +95,6 @@ public static void main(String[] args) throws FileNotFoundException { BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { rows.set(rowDataset.toJSON().collectAsList()); - //rowDataset.show(false); }; final DPLExecutor dplExecutor; @@ -110,24 +114,7 @@ public static void main(String[] args) throws FileNotFoundException { final DPLExecutorResult executorResult = dplExecutor .interpret(batchHandler, sparkSession, queryId, noteId, paragraphId, lines); - /* - final InterpreterResult.Code code; - if (executorResult.code().equals(DPLExecutorResult.Code.SUCCESS)) { - code = Code.SUCCESS; - } else if (executorResult.code().equals(DPLExecutorResult.Code.INCOMPLETE)) { - code = DPLExecutorResult.Code.INCOMPLETE; - } else if (executorResult.code().equals(DPLExecutorResult.Code.KEEP_PREVIOUS_RESULT)) { - code = Code.KEEP_PREVIOUS_RESULT; - } else { - code = Code.ERROR; - } - */ - - System.out - .println( - "executorResult code <" + executorResult.code() + "> message <" + executorResult.message() - + ">" - ); + LOGGER.info("executorResult code <{}> message <{}>", executorResult.code(), executorResult.message()); for (String string : rows.get()) { System.out.println(string); From 384279f1c57f9d90dc557272bb4d3ebef4843d9c Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 14:06:37 +0200 Subject: [PATCH 11/15] add final clauses --- src/main/java/com/teragrep/pth_16/Entry.java | 24 +++++++-------- .../pth_16/InterpreterSettingsConfig.java | 29 ++++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index 5e1c9c9..b8ba730 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -65,19 +65,19 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; -public class Entry { +public final class Entry { - public static void main(String[] args) throws FileNotFoundException { + public static void main(final String[] args) throws FileNotFoundException { - Logger LOGGER = LoggerFactory.getLogger(Entry.class); + final Logger LOGGER = LoggerFactory.getLogger(Entry.class); - File configFile = new File("interpreter.json"); + final File configFile = new File("interpreter.json"); - InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); + final InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); - Config config = interpreterSettingsConfig.getConfig(); + final Config config = interpreterSettingsConfig.getConfig(); - String applicationName = "com.teragrep.pth_16.Entry"; + final String applicationName = "com.teragrep.pth_16.Entry"; final String lines; if (args.length > 0) { @@ -87,13 +87,13 @@ public static void main(String[] args) throws FileNotFoundException { lines = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; } - String queryId = UUID.randomUUID().toString(); - String noteId = "pth_16-notebook-" + UUID.randomUUID(); - String paragraphId = "pth_16-paragraph-" + UUID.randomUUID(); + final String queryId = UUID.randomUUID().toString(); + final String noteId = "pth_16-notebook-" + UUID.randomUUID(); + final String paragraphId = "pth_16-paragraph-" + UUID.randomUUID(); final AtomicReference> rows = new AtomicReference<>(new ArrayList<>()); - BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { + final BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { rows.set(rowDataset.toJSON().collectAsList()); }; @@ -108,7 +108,7 @@ public static void main(String[] args) throws FileNotFoundException { throw new RuntimeException("Error initializing DPLExecutor implementation", e); } - SparkSession sparkSession = SparkSession.builder().appName(applicationName).getOrCreate(); + final SparkSession sparkSession = SparkSession.builder().appName(applicationName).getOrCreate(); try { final DPLExecutorResult executorResult = dplExecutor diff --git a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java index 9b38df4..77d8621 100644 --- a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java +++ b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java @@ -60,11 +60,11 @@ import java.io.FileNotFoundException; import java.util.Map; -public class InterpreterSettingsConfig { +public final class InterpreterSettingsConfig { private final File configFile; - public InterpreterSettingsConfig(File configFile) { + public InterpreterSettingsConfig(final File configFile) { this.configFile = configFile; } @@ -73,29 +73,29 @@ public Config getConfig() throws FileNotFoundException { Config config = ConfigFactory.empty(); try (JsonReader reader = Json.createReader(new FileReader(configFile))) { - JsonObject jsonObject = reader.readObject(); + final JsonObject jsonObject = reader.readObject(); if (!jsonObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException("config does not contain a JSON object"); } - String interpreterSettingsKey = "interpreterSettings"; + final String interpreterSettingsKey = "interpreterSettings"; if (!jsonObject.containsKey(interpreterSettingsKey)) { throw new IllegalArgumentException( "config does not contain a JSON object with key <" + interpreterSettingsKey + ">" ); } - JsonObject interpreterSettings = jsonObject.getJsonObject(interpreterSettingsKey); + final JsonObject interpreterSettings = jsonObject.getJsonObject(interpreterSettingsKey); - String sparkKey = "spark"; + final String sparkKey = "spark"; if (!interpreterSettings.containsKey(sparkKey)) { throw new IllegalArgumentException( "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + ">" ); } - JsonObject sparkObject = interpreterSettings.getJsonObject(sparkKey); + final JsonObject sparkObject = interpreterSettings.getJsonObject(sparkKey); if (!sparkObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException( @@ -104,7 +104,7 @@ public Config getConfig() throws FileNotFoundException { ); } - String propertieskey = "properties"; + final String propertieskey = "properties"; if (!sparkObject.containsKey(propertieskey)) { throw new IllegalArgumentException( "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey @@ -112,7 +112,7 @@ public Config getConfig() throws FileNotFoundException { ); } - JsonObject propertiesObject = sparkObject.getJsonObject(propertieskey); + final JsonObject propertiesObject = sparkObject.getJsonObject(propertieskey); if (!propertiesObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException( @@ -123,24 +123,25 @@ public Config getConfig() throws FileNotFoundException { for (Map.Entry entry : propertiesObject.entrySet()) { if (entry.getKey().startsWith("dpl.") || entry.getKey().startsWith("fs.s3a.")) { - String key = entry.getKey(); + final String key = entry.getKey(); - JsonValue entryValue = entry.getValue(); + final JsonValue entryValue = entry.getValue(); if (!entryValue.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException("json key <[" + key + "]> does not refer to a json object"); } - JsonObject entryValueObject = entryValue.asJsonObject(); + final JsonObject entryValueObject = entryValue.asJsonObject(); - String valueKey = "value"; + final String valueKey = "value"; if (!entryValueObject.containsKey(valueKey)) { throw new IllegalArgumentException( "json key <[" + key + "]> does not refer to a json object with a key named <" + valueKey + ">" ); } - JsonValue jsonValue = entryValueObject.get(valueKey); + + final JsonValue jsonValue = entryValueObject.get(valueKey); switch (jsonValue.getValueType()) { case STRING: From b64007c3d0d020cd62f763ace71956f34228c96c Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 14:10:29 +0200 Subject: [PATCH 12/15] remove central deploy --- pom.xml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/pom.xml b/pom.xml index 0a53155..91d30d5 100644 --- a/pom.xml +++ b/pom.xml @@ -533,56 +533,6 @@ - - publish-maven-central - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.5.0 - true - - central-sonatype-org - true - true - published - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - - sign - - verify - - - --pinentry-mode - loopback - - - - - - - - publish-github-packages From 79926b489ce0210d967a82cd09bd8ad83b414c0d Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Tue, 27 Jan 2026 14:39:34 +0200 Subject: [PATCH 13/15] rename getConfig to config --- src/main/java/com/teragrep/pth_16/Entry.java | 2 +- .../java/com/teragrep/pth_16/InterpreterSettingsConfig.java | 2 +- .../java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index b8ba730..5870763 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -75,7 +75,7 @@ public static void main(final String[] args) throws FileNotFoundException { final InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); - final Config config = interpreterSettingsConfig.getConfig(); + final Config config = interpreterSettingsConfig.config(); final String applicationName = "com.teragrep.pth_16.Entry"; diff --git a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java index 77d8621..7e0f80f 100644 --- a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java +++ b/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java @@ -68,7 +68,7 @@ public InterpreterSettingsConfig(final File configFile) { this.configFile = configFile; } - public Config getConfig() throws FileNotFoundException { + public Config config() throws FileNotFoundException { Config config = ConfigFactory.empty(); diff --git a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java index f79e425..9fe04d1 100644 --- a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java +++ b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java @@ -60,7 +60,7 @@ public void loadNFilterZep01InterpreterSettings() { ); Assertions.assertDoesNotThrow(() -> { - Config config = interpreterSettingsConfig.getConfig(); + Config config = interpreterSettingsConfig.config(); Assertions.assertFalse(config.getBoolean("dpl.pth_06.sql.log.enabled")); From b8b2e1eab6f2931de89198cedff3db305e61e480 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Thu, 29 Jan 2026 18:22:27 +0200 Subject: [PATCH 14/15] more objects --- src/main/java/com/teragrep/pth_16/Entry.java | 93 ++--- .../com/teragrep/pth_16/FilterableConfig.java | 55 +++ .../teragrep/pth_16/FilterableConfigImpl.java | 362 ++++++++++++++++++ .../pth_16/InterpreterProperties.java | 112 ++++++ ...gsConfig.java => InterpreterSettings.java} | 83 +--- src/main/java/com/teragrep/pth_16/Query.java | 135 +++++++ .../pth_16/InterpreterSettingsConfigTest.java | 13 +- 7 files changed, 731 insertions(+), 122 deletions(-) create mode 100644 src/main/java/com/teragrep/pth_16/FilterableConfig.java create mode 100644 src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java create mode 100644 src/main/java/com/teragrep/pth_16/InterpreterProperties.java rename src/main/java/com/teragrep/pth_16/{InterpreterSettingsConfig.java => InterpreterSettings.java} (53%) create mode 100644 src/main/java/com/teragrep/pth_16/Query.java diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index 5870763..338a4c9 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -45,83 +45,74 @@ */ package com.teragrep.pth_16; -import com.teragrep.pth_15.DPLExecutor; -import com.teragrep.pth_15.DPLExecutorFactory; -import com.teragrep.pth_15.DPLExecutorResult; import com.typesafe.config.Config; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; +import com.typesafe.config.ConfigValue; +import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; +import java.util.Map; +import java.util.Set; public final class Entry { public static void main(final String[] args) throws FileNotFoundException { - final Logger LOGGER = LoggerFactory.getLogger(Entry.class); - final File configFile = new File("interpreter.json"); - final InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig(configFile); + final InterpreterSettings interpreterSettings = new InterpreterSettings(configFile); - final Config config = interpreterSettingsConfig.config(); + final InterpreterProperties sparkProperties = interpreterSettings.interpreterProperties("spark"); - final String applicationName = "com.teragrep.pth_16.Entry"; + final Config sparkInterpterConfig = sparkProperties.config(); - final String lines; - if (args.length > 0) { - lines = args[0]; - } - else { - lines = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; - } + final FilterableConfig filterableConfig = new FilterableConfigImpl(sparkInterpterConfig); + + // dpl + final Set dplConfigFilters = new HashSet<>(); + dplConfigFilters.add("dpl."); + dplConfigFilters.add("fs.s3a."); + final Config dplConfig = filterableConfig.startsWith(dplConfigFilters); - final String queryId = UUID.randomUUID().toString(); - final String noteId = "pth_16-notebook-" + UUID.randomUUID(); - final String paragraphId = "pth_16-paragraph-" + UUID.randomUUID(); + // spark + final Set sparkConfigFilters = new HashSet<>(); + sparkConfigFilters.add("spark."); + final Config sparkConfig = filterableConfig.startsWith(sparkConfigFilters); + final SparkConf sparkConf = new SparkConf(); - final AtomicReference> rows = new AtomicReference<>(new ArrayList<>()); + for (Map.Entry sparkConfigEntry : sparkConfig.entrySet()) { + String key = sparkConfigEntry.getKey(); + // ignore spark-submit configs such as spark.submit.deployMode + if (!key.startsWith("spark.submit.")) { + sparkConf.set(key, sparkConfigEntry.getValue().unwrapped().toString()); + } + } - final BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { - rows.set(rowDataset.toJSON().collectAsList()); - }; + final String applicationName = "com.teragrep.pth_16.Entry"; - final DPLExecutor dplExecutor; - try { - dplExecutor = new DPLExecutorFactory("com.teragrep.pth_10.executor.DPLExecutorImpl", config).create(); + final String prompt; + if (args.length > 0) { + prompt = args[0]; } - catch ( - ClassNotFoundException | NoSuchMethodException | InvocationTargetException | InstantiationException - | IllegalAccessException e - ) { - throw new RuntimeException("Error initializing DPLExecutor implementation", e); + else { + prompt = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; } - final SparkSession sparkSession = SparkSession.builder().appName(applicationName).getOrCreate(); + final SparkSession sparkSession = SparkSession + .builder() + .config(sparkConf) + .appName(applicationName) + .getOrCreate(); - try { - final DPLExecutorResult executorResult = dplExecutor - .interpret(batchHandler, sparkSession, queryId, noteId, paragraphId, lines); + final Query query = new Query(sparkSession, dplConfig, prompt); - LOGGER.info("executorResult code <{}> message <{}>", executorResult.code(), executorResult.message()); + final List rows = query.run(); - for (String string : rows.get()) { - System.out.println(string); - } - } - catch (TimeoutException e) { - throw new RuntimeException(e); + for (String string : rows) { + System.out.println(string); } } diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfig.java b/src/main/java/com/teragrep/pth_16/FilterableConfig.java new file mode 100644 index 0000000..7151f2f --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/FilterableConfig.java @@ -0,0 +1,55 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; + +import java.util.Set; + +public interface FilterableConfig extends Config { + + public abstract Config startsWith(Set filters); +} diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java new file mode 100644 index 0000000..9e1f9da --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java @@ -0,0 +1,362 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.*; + +import java.time.Duration; +import java.time.Period; +import java.time.temporal.TemporalAmount; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class FilterableConfigImpl implements FilterableConfig { + + private final Config config; + + public FilterableConfigImpl(final Config config) { + this.config = config; + } + + @Override + public ConfigObject root() { + return config.root(); + } + + @Override + public ConfigOrigin origin() { + return config.origin(); + } + + @Override + public Config withFallback(final ConfigMergeable other) { + return config.withFallback(other); + } + + @Override + public Config resolve() { + return config.resolve(); + } + + @Override + public Config resolve(final ConfigResolveOptions options) { + return config.resolve(options); + } + + @Override + public boolean isResolved() { + return config.isResolved(); + } + + @Override + public Config resolveWith(final Config source) { + return config.resolveWith(source); + } + + @Override + public Config resolveWith(final Config source, final ConfigResolveOptions options) { + return config.resolveWith(source, options); + } + + @Override + public void checkValid(final Config reference, final String ... restrictToPaths) { + config.checkValid(reference, restrictToPaths); + } + + @Override + public boolean hasPath(final String path) { + return config.hasPath(path); + } + + @Override + public boolean hasPathOrNull(final String path) { + return config.hasPathOrNull(path); + } + + @Override + public boolean isEmpty() { + return config.isEmpty(); + } + + @Override + public Set> entrySet() { + return config.entrySet(); + } + + @Override + public boolean getIsNull(final String path) { + return config.getIsNull(path); + } + + @Override + public boolean getBoolean(final String path) { + return config.getBoolean(path); + } + + @Override + public Number getNumber(final String path) { + return config.getNumber(path); + } + + @Override + public int getInt(final String path) { + return config.getInt(path); + } + + @Override + public long getLong(final String path) { + return config.getLong(path); + } + + @Override + public double getDouble(final String path) { + return config.getDouble(path); + } + + @Override + public String getString(final String path) { + return config.getString(path); + } + + @Override + public > T getEnum(final Class enumClass, final String path) { + return config.getEnum(enumClass, path); + } + + @Override + public ConfigObject getObject(final String path) { + return config.getObject(path); + } + + @Override + public Config getConfig(final String path) { + return config.getConfig(path); + } + + @Override + public Object getAnyRef(final String path) { + return config.getAnyRef(path); + } + + @Override + public ConfigValue getValue(final String path) { + return config.getValue(path); + } + + @Override + public Long getBytes(final String path) { + return config.getBytes(path); + } + + @Override + public ConfigMemorySize getMemorySize(final String path) { + return config.getMemorySize(path); + } + + @Deprecated + @Override + public Long getMilliseconds(final String path) { + return config.getMilliseconds(path); + } + + @Deprecated + @Override + public Long getNanoseconds(final String path) { + return config.getNanoseconds(path); + } + + @Override + public long getDuration(final String path, final TimeUnit unit) { + return config.getDuration(path, unit); + } + + @Override + public Duration getDuration(final String path) { + return config.getDuration(path); + } + + @Override + public Period getPeriod(final String path) { + return config.getPeriod(path); + } + + @Override + public TemporalAmount getTemporal(final String path) { + return config.getTemporal(path); + } + + @Override + public ConfigList getList(final String path) { + return config.getList(path); + } + + @Override + public List getBooleanList(final String path) { + return config.getBooleanList(path); + } + + @Override + public List getNumberList(final String path) { + return config.getNumberList(path); + } + + @Override + public List getIntList(final String path) { + return config.getIntList(path); + } + + @Override + public List getLongList(final String path) { + return config.getLongList(path); + } + + @Override + public List getDoubleList(final String path) { + return config.getDoubleList(path); + } + + @Override + public List getStringList(final String path) { + return config.getStringList(path); + } + + @Override + public > List getEnumList(final Class enumClass, final String path) { + return config.getEnumList(enumClass, path); + } + + @Override + public List getObjectList(final String path) { + return config.getObjectList(path); + } + + @Override + public List getConfigList(final String path) { + return config.getConfigList(path); + } + + @Override + public List getAnyRefList(final String path) { + return config.getAnyRefList(path); + } + + @Override + public List getBytesList(final String path) { + return config.getBytesList(path); + } + + @Override + public List getMemorySizeList(final String path) { + return config.getMemorySizeList(path); + } + + @Deprecated + @Override + public List getMillisecondsList(final String path) { + return config.getMillisecondsList(path); + } + + @Deprecated + @Override + public List getNanosecondsList(final String path) { + return config.getNanosecondsList(path); + } + + @Override + public List getDurationList(final String path, final TimeUnit unit) { + return config.getDurationList(path, unit); + } + + @Override + public List getDurationList(final String path) { + return config.getDurationList(path); + } + + @Override + public Config withOnlyPath(final String path) { + return config.withOnlyPath(path); + } + + @Override + public Config withoutPath(final String path) { + return config.withoutPath(path); + } + + @Override + public Config atPath(final String path) { + return config.atPath(path); + } + + @Override + public Config atKey(final String key) { + return config.atKey(key); + } + + @Override + public Config withValue(final String path, final ConfigValue value) { + return config.withValue(path, value); + } + + @Override + public Config startsWith(final Set filters) { + Config filteredConfig = ConfigFactory.empty(); + + for (Map.Entry entry : config.entrySet()) { + final String key = entry.getKey(); + + for (final String filter : filters) { + if (key.startsWith(filter)) { + filteredConfig = filteredConfig.withValue(key, entry.getValue()); + } + } + } + return filteredConfig; + + } + +} diff --git a/src/main/java/com/teragrep/pth_16/InterpreterProperties.java b/src/main/java/com/teragrep/pth_16/InterpreterProperties.java new file mode 100644 index 0000000..20d7477 --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/InterpreterProperties.java @@ -0,0 +1,112 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValueFactory; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + +import java.util.Map; + +public class InterpreterProperties { + + private final JsonObject propertiesJson; + + public InterpreterProperties(final JsonObject propertiesJson) { + this.propertiesJson = propertiesJson; + } + + Config config() { + Config config = ConfigFactory.empty(); + for (Map.Entry entry : propertiesJson.entrySet()) { + final String key = entry.getKey(); + + final JsonValue entryValue = entry.getValue(); + + if (!entryValue.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException("json key <[" + key + "]> does not refer to a json object"); + } + + final JsonObject entryValueObject = entryValue.asJsonObject(); + + final String valueKey = "value"; + if (!entryValueObject.containsKey(valueKey)) { + throw new IllegalArgumentException( + "json key <[" + key + "]> does not refer to a json object with a key named <" + valueKey + ">" + ); + } + + final JsonValue jsonValue = entryValueObject.get(valueKey); + + switch (jsonValue.getValueType()) { + case STRING: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getString("value"))); + break; + case NULL: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(null)); + break; + case TRUE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(true)); + break; + case FALSE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(false)); + break; + case NUMBER: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getInt("value"))); + break; + default: + throw new IllegalArgumentException( + "json key <[" + key + "]> has object with key <" + valueKey + + "> that has unsupported value type" + jsonValue.getValueType() + ); + } + } + return config; + } + +} diff --git a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java b/src/main/java/com/teragrep/pth_16/InterpreterSettings.java similarity index 53% rename from src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java rename to src/main/java/com/teragrep/pth_16/InterpreterSettings.java index 7e0f80f..d49a4ac 100644 --- a/src/main/java/com/teragrep/pth_16/InterpreterSettingsConfig.java +++ b/src/main/java/com/teragrep/pth_16/InterpreterSettings.java @@ -45,12 +45,8 @@ */ package com.teragrep.pth_16; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; - import java.io.File; -import com.typesafe.config.ConfigValueFactory; import jakarta.json.Json; import jakarta.json.JsonObject; import jakarta.json.JsonReader; @@ -58,19 +54,16 @@ import java.io.FileReader; import java.io.FileNotFoundException; -import java.util.Map; -public final class InterpreterSettingsConfig { +public final class InterpreterSettings { private final File configFile; - public InterpreterSettingsConfig(final File configFile) { + public InterpreterSettings(final File configFile) { this.configFile = configFile; } - public Config config() throws FileNotFoundException { - - Config config = ConfigFactory.empty(); + public InterpreterProperties interpreterProperties(final String interpreterName) throws FileNotFoundException { try (JsonReader reader = Json.createReader(new FileReader(configFile))) { final JsonObject jsonObject = reader.readObject(); @@ -88,88 +81,40 @@ public Config config() throws FileNotFoundException { final JsonObject interpreterSettings = jsonObject.getJsonObject(interpreterSettingsKey); - final String sparkKey = "spark"; - if (!interpreterSettings.containsKey(sparkKey)) { + if (!interpreterSettings.containsKey(interpreterName)) { throw new IllegalArgumentException( - "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + ">" + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + + ">" ); } - final JsonObject sparkObject = interpreterSettings.getJsonObject(sparkKey); + final JsonObject interpreterObject = interpreterSettings.getJsonObject(interpreterName); - if (!sparkObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + if (!interpreterObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException( - "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + "> as a JSON object" ); } final String propertieskey = "properties"; - if (!sparkObject.containsKey(propertieskey)) { + if (!interpreterObject.containsKey(propertieskey)) { throw new IllegalArgumentException( - "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + "> with key <" + propertieskey + ">" ); } - final JsonObject propertiesObject = sparkObject.getJsonObject(propertieskey); + final JsonObject propertiesObject = interpreterObject.getJsonObject(propertieskey); if (!propertiesObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { throw new IllegalArgumentException( - "config does not contain a <" + interpreterSettingsKey + "> object with key <" + sparkKey + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + "> with key <" + propertieskey + "> as a JSON object" ); } - for (Map.Entry entry : propertiesObject.entrySet()) { - if (entry.getKey().startsWith("dpl.") || entry.getKey().startsWith("fs.s3a.")) { - final String key = entry.getKey(); - - final JsonValue entryValue = entry.getValue(); - - if (!entryValue.getValueType().equals(JsonValue.ValueType.OBJECT)) { - throw new IllegalArgumentException("json key <[" + key + "]> does not refer to a json object"); - } - - final JsonObject entryValueObject = entryValue.asJsonObject(); - - final String valueKey = "value"; - if (!entryValueObject.containsKey(valueKey)) { - throw new IllegalArgumentException( - "json key <[" + key + "]> does not refer to a json object with a key named <" + valueKey - + ">" - ); - } - - final JsonValue jsonValue = entryValueObject.get(valueKey); - - switch (jsonValue.getValueType()) { - case STRING: - config = config - .withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getString("value"))); - break; - case NULL: - config = config.withValue(key, ConfigValueFactory.fromAnyRef(null)); - break; - case TRUE: - config = config.withValue(key, ConfigValueFactory.fromAnyRef(true)); - break; - case FALSE: - config = config.withValue(key, ConfigValueFactory.fromAnyRef(false)); - break; - case NUMBER: - config = config - .withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getInt("value"))); - break; - default: - throw new IllegalArgumentException( - "json key <[" + key + "]> has object with key <" + valueKey - + "> that has unsupported value type" + jsonValue.getValueType() - ); - } - } - } + return new InterpreterProperties(propertiesObject); } - return config; } } diff --git a/src/main/java/com/teragrep/pth_16/Query.java b/src/main/java/com/teragrep/pth_16/Query.java new file mode 100644 index 0000000..b0e5ddd --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/Query.java @@ -0,0 +1,135 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.teragrep.pth_15.DPLExecutor; +import com.teragrep.pth_15.DPLExecutorFactory; +import com.teragrep.pth_15.DPLExecutorResult; +import com.typesafe.config.Config; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + +public class Query { + + private final Logger LOGGER = LoggerFactory.getLogger(Query.class); + + private final SparkSession sparkSession; + private final Config dplConfig; + private final String queryId; + private final String noteId; + private final String paragraphId; + private final String prompt; + + public Query(final SparkSession sparkSession, final Config dplConfig, final String prompt) { + this( + sparkSession, + dplConfig, + UUID.randomUUID().toString(), + "pth_16-notebook-" + UUID.randomUUID(), + "pth_16-paragraph-" + UUID.randomUUID(), + prompt + ); + } + + public Query( + final SparkSession sparkSession, + final Config dplConfig, + final String queryId, + final String noteId, + final String paragraphId, + final String prompt + ) { + this.sparkSession = sparkSession; + this.dplConfig = dplConfig; + this.queryId = queryId; + this.noteId = noteId; + this.paragraphId = paragraphId; + this.prompt = prompt; + } + + public List run() { + final AtomicReference> rows = new AtomicReference<>(new ArrayList<>()); + + final BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { + rows.set(rowDataset.toJSON().collectAsList()); + }; + + final DPLExecutor dplExecutor; + try { + dplExecutor = new DPLExecutorFactory("com.teragrep.pth_10.executor.DPLExecutorImpl", dplConfig).create(); + } + catch ( + ClassNotFoundException | NoSuchMethodException | InvocationTargetException | InstantiationException + | IllegalAccessException e + ) { + throw new RuntimeException("Error initializing DPLExecutor implementation", e); + } + + try { + final DPLExecutorResult executorResult = dplExecutor + .interpret(batchHandler, sparkSession, queryId, noteId, paragraphId, prompt); + + LOGGER.info("executorResult code <{}> message <{}>", executorResult.code(), executorResult.message()); + + } + catch (TimeoutException e) { + throw new RuntimeException(e); + } + + return rows.get(); + } +} diff --git a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java index 9fe04d1..e8b80df 100644 --- a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java +++ b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java @@ -50,17 +50,26 @@ import org.junit.jupiter.api.Test; import java.io.File; +import java.util.HashSet; +import java.util.Set; public class InterpreterSettingsConfigTest { @Test public void loadNFilterZep01InterpreterSettings() { - InterpreterSettingsConfig interpreterSettingsConfig = new InterpreterSettingsConfig( + InterpreterSettings interpreterSettings = new InterpreterSettings( new File("src/test/resources/interpreter.json") ); Assertions.assertDoesNotThrow(() -> { - Config config = interpreterSettingsConfig.config(); + InterpreterProperties sparkProperties = interpreterSettings.interpreterProperties("spark"); + Config sparkConfig = sparkProperties.config(); + FilterableConfig filterableSparkConfig = new FilterableConfigImpl(sparkConfig); + + Set dplConfigFilters = new HashSet<>(); + dplConfigFilters.add("dpl."); + dplConfigFilters.add("fs.s3a."); + Config config = filterableSparkConfig.startsWith(dplConfigFilters); Assertions.assertFalse(config.getBoolean("dpl.pth_06.sql.log.enabled")); From 70906f2019c34cad0ba86438c932152977b4fa63 Mon Sep 17 00:00:00 2001 From: Motoko Kusanagi Date: Fri, 30 Jan 2026 09:14:37 +0200 Subject: [PATCH 15/15] add startsWith(String ... filters); to FilterableConfig for more handy usage --- src/main/java/com/teragrep/pth_16/Entry.java | 25 ++++++------------- .../com/teragrep/pth_16/FilterableConfig.java | 2 ++ .../teragrep/pth_16/FilterableConfigImpl.java | 9 ++++--- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java index 338a4c9..340fd39 100644 --- a/src/main/java/com/teragrep/pth_16/Entry.java +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -52,10 +52,8 @@ import java.io.File; import java.io.FileNotFoundException; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; public final class Entry { @@ -71,16 +69,7 @@ public static void main(final String[] args) throws FileNotFoundException { final FilterableConfig filterableConfig = new FilterableConfigImpl(sparkInterpterConfig); - // dpl - final Set dplConfigFilters = new HashSet<>(); - dplConfigFilters.add("dpl."); - dplConfigFilters.add("fs.s3a."); - final Config dplConfig = filterableConfig.startsWith(dplConfigFilters); - - // spark - final Set sparkConfigFilters = new HashSet<>(); - sparkConfigFilters.add("spark."); - final Config sparkConfig = filterableConfig.startsWith(sparkConfigFilters); + final Config sparkConfig = filterableConfig.startsWith("spark."); final SparkConf sparkConf = new SparkConf(); for (Map.Entry sparkConfigEntry : sparkConfig.entrySet()) { @@ -93,6 +82,12 @@ public static void main(final String[] args) throws FileNotFoundException { final String applicationName = "com.teragrep.pth_16.Entry"; + final SparkSession sparkSession = SparkSession + .builder() + .config(sparkConf) + .appName(applicationName) + .getOrCreate(); + final String prompt; if (args.length > 0) { prompt = args[0]; @@ -101,11 +96,7 @@ public static void main(final String[] args) throws FileNotFoundException { prompt = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; } - final SparkSession sparkSession = SparkSession - .builder() - .config(sparkConf) - .appName(applicationName) - .getOrCreate(); + final Config dplConfig = filterableConfig.startsWith("dpl.", "fs.s3a."); final Query query = new Query(sparkSession, dplConfig, prompt); diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfig.java b/src/main/java/com/teragrep/pth_16/FilterableConfig.java index 7151f2f..0df201b 100644 --- a/src/main/java/com/teragrep/pth_16/FilterableConfig.java +++ b/src/main/java/com/teragrep/pth_16/FilterableConfig.java @@ -52,4 +52,6 @@ public interface FilterableConfig extends Config { public abstract Config startsWith(Set filters); + + public abstract Config startsWith(String ... filters); } diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java index 9e1f9da..3c00084 100644 --- a/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java +++ b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java @@ -50,9 +50,7 @@ import java.time.Duration; import java.time.Period; import java.time.temporal.TemporalAmount; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; public class FilterableConfigImpl implements FilterableConfig { @@ -359,4 +357,9 @@ public Config startsWith(final Set filters) { } + @Override + public Config startsWith(final String ... filters) { + Set filterSet = new HashSet<>(Arrays.asList(filters)); + return startsWith(filterSet); + } }