diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 5e14671..db35714 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -4,7 +4,7 @@ contact_links: url: https://github.com/teragrep/doc_01/issues/new?template=doc-issue-report.md about: Problems with Teragrep documentation - name: Ask a question or get support - url: https://github.com/teragrep/repo-template/discussions + url: https://github.com/teragrep/pth_16/discussions about: Ask a question or request support - name: Report vulnerability url: https://github.com/teragrep/teragrep/security/advisories/new diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4774b04 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/.settings/* +/.vscode/* +/.flattened-pom.xml +/target/* +/.classpath +/.factorypath +/.project +/.idea/ +/*.iml diff --git a/README.adoc b/README.adoc index 0f563cc..a6ea6ab 100644 --- a/README.adoc +++ b/README.adoc @@ -1,17 +1,11 @@ -// Before publishing your new repository: -// 1. Write the readme file -// 2. Update the issues link in Contributing section in the readme file -// 3. Update the discussion link in config.yml file in .github/ISSUE_TEMPLATE directory -= repo-template += Teragrep DPL Command-Line Executor (pth_16) -// Add a short description of your project. Tell what your project does and what it's used for. - -This is a template repository for Teragrep organization. +Teragrep DPL Command-Line Executor is an utility to run DPL queries from command-line. == Features -// List your project's features +- Allows running Teragrep queries from command-line. == Documentation @@ -19,17 +13,86 @@ See the official documentation on https://docs.teragrep.com[docs.teragrep.com]. == Limitations -// If your project has limitations, please list them. Otherwise remove this section. +Uses spark-submit. == How to [compile/use/implement] -// add instructions how people can start to use your project +=== Usage + +Example without arguments + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar +---- + +Results into + +[source,json] +---- +{"_time":"2026-01-27T13:37:00.000+02:00","_raw":"Welcome to Teragrep®"} +---- + +Custom query + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar \ + '| makeresults count=1 | eval _raw="this is a custom query"' + +---- + +Results into + +[source,json] +---- +{"_time":"2026-01-27T13:37:54.000+02:00","_raw":"this is a custom query"} +---- + +Results can be piped into a file + +[source,shell] +---- +JAVA_HOME=/usr/lib/jvm/java-11 /opt/teragrep/spk_02/bin/spark-submit \ + --verbose \ + --deploy-mode client \ + --master yarn \ + --executor-memory 5G \ + --total-executor-cores 8 \ + --jars /opt/teragrep/pth_10/lib/pth_10-shaded.jar,/opt/teragrep/spk_03/lib/*.jar \ + --files interpreter.json pth_16-jar-with-dependencies.jar \ + '| makeresults count=1 | eval _raw="this is a custom query"' | tee output.json +---- + +Results into + +[source,json] +---- +{ + "_time": "2026-01-27T13:54:24.000+02:00", + "_raw": "this is a custom query" +} +---- == Contributing // Change the repository name in the issues link to match with your project's name -You can involve yourself with our project by https://github.com/teragrep/repo-template/issues/new/choose[opening an issue] or submitting a pull request. +You can involve yourself with our project by https://github.com/teragrep/pth_16/issues/new/choose[opening an issue] or submitting a pull request. Contribution requirements: diff --git a/eclipse-java-formatter.xml b/eclipse-java-formatter.xml new file mode 100644 index 0000000..53b9f2a --- /dev/null +++ b/eclipse-java-formatter.xml @@ -0,0 +1,450 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/license-header b/license-header new file mode 100644 index 0000000..51ba6ec --- /dev/null +++ b/license-header @@ -0,0 +1,45 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..91d30d5 --- /dev/null +++ b/pom.xml @@ -0,0 +1,571 @@ + + + 4.0.0 + com.teragrep + pth_16 + ${revision}${sha1}${changelist} + jar + pth_16 + Teragrep DPL Command-Line Executor + https://teragrep.com + + + GNU Affero General Public License v3.0 + https://www.gnu.org/licenses/agpl-3.0.txt + + + + + Mikko Kortelainen + 1@teragrep.com + Teragrep + https://teragrep.com + + + + scm:git:https://github.com/teragrep/pth_16.git + scm:git:git@github.com:teragrep/pth_16.git + https://github.com/teragrep/pth_16/tree/master + + + -SNAPSHOT + 1.8 + 2.3.0 + 1.8 + 1.8 + 4.2.0 + UTF-8 + 0.0.1 + + + + + com.teragrep + pth_15 + 2.0.0 + + + org.apache.spark + spark-sql_2.12 + 3.5.7 + provided + + + jakarta.json + jakarta.json-api + 2.1.3 + + + org.eclipse.parsson + parsson + 1.1.5 + runtime + + + org.junit.jupiter + junit-jupiter-engine + 5.4.0-RC1 + test + + + org.junit.platform + junit-platform-launcher + 1.4.0-RC1 + test + + + org.junit.jupiter + junit-jupiter-api + 5.4.0-RC1 + test + + + nl.jqno.equalsverifier + equalsverifier + 3.16.1 + test + + + + ${project.basedir}/target + pth_16 + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + jar-with-dependencies + + + + com.teragrep.pth_16.Entry + + + + + + make-assembly + + single + + package + + + + + org.apache.maven.plugins + maven-source-plugin + 3.3.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.6.3 + + + attach-javadocs + + jar + + + 8 + false + none + + + startuml + + startuml + + + enduml + + enduml + + + class + + class + + + brief + + brief + + + responsibilities + + responsibilities + + + collaborators + + collaborators + + + + + + + + org.apache.rat + apache-rat-plugin + 0.15 + false + + false + + + Also allow the license url to be https. + + GNU Affero General Public License + + + + true + false + + + .git/** + .gitattributes + .gitignore + .gitmodules + + .github/workflows/*.yml + .github/workflows/*.yaml + .github/ISSUE_TEMPLATE/* + .github/pull_request_template.md + + README.adoc + README.md + + Doxyfile + + + + + + check + + test + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.30.0 + + + + ${project.basedir}/eclipse-java-formatter.xml + 4.10.0 + + + + ${project.basedir}/license-header + + + + + + UTF-8 + \n + true + false + 2 + recommended_2008_06 + true + true + true + + + + + + .gitattributes + .gitignore + + + + + true + 4 + + + + + + + + check + + compile + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + bom + + + + flatten.clean + + clean + + clean + + + flatten + + flatten + + process-resources + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + -Xlint:all + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + all + true + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.5.0 + + + + scan-errors + + check + + process-classes + + error + true + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + scan-warnings + + check + + process-classes + + warning + true + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + publish-github-packages + + + github + GitHub Packages + https://maven.pkg.github.com/teragrep/pth_16 + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + + sign + + verify + + + --pinentry-mode + loopback + + + + + + + + + + diff --git a/src/main/java/com/teragrep/pth_16/Entry.java b/src/main/java/com/teragrep/pth_16/Entry.java new file mode 100644 index 0000000..338a4c9 --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/Entry.java @@ -0,0 +1,119 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValue; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public final class Entry { + + public static void main(final String[] args) throws FileNotFoundException { + + final File configFile = new File("interpreter.json"); + + final InterpreterSettings interpreterSettings = new InterpreterSettings(configFile); + + final InterpreterProperties sparkProperties = interpreterSettings.interpreterProperties("spark"); + + final Config sparkInterpterConfig = sparkProperties.config(); + + final FilterableConfig filterableConfig = new FilterableConfigImpl(sparkInterpterConfig); + + // dpl + final Set dplConfigFilters = new HashSet<>(); + dplConfigFilters.add("dpl."); + dplConfigFilters.add("fs.s3a."); + final Config dplConfig = filterableConfig.startsWith(dplConfigFilters); + + // spark + final Set sparkConfigFilters = new HashSet<>(); + sparkConfigFilters.add("spark."); + final Config sparkConfig = filterableConfig.startsWith(sparkConfigFilters); + final SparkConf sparkConf = new SparkConf(); + + for (Map.Entry sparkConfigEntry : sparkConfig.entrySet()) { + String key = sparkConfigEntry.getKey(); + // ignore spark-submit configs such as spark.submit.deployMode + if (!key.startsWith("spark.submit.")) { + sparkConf.set(key, sparkConfigEntry.getValue().unwrapped().toString()); + } + } + + final String applicationName = "com.teragrep.pth_16.Entry"; + + final String prompt; + if (args.length > 0) { + prompt = args[0]; + } + else { + prompt = "| makeresults count=1 | eval _raw=\"Welcome to Teragrep®\""; + } + + final SparkSession sparkSession = SparkSession + .builder() + .config(sparkConf) + .appName(applicationName) + .getOrCreate(); + + final Query query = new Query(sparkSession, dplConfig, prompt); + + final List rows = query.run(); + + for (String string : rows) { + System.out.println(string); + } + } + +} diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfig.java b/src/main/java/com/teragrep/pth_16/FilterableConfig.java new file mode 100644 index 0000000..7151f2f --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/FilterableConfig.java @@ -0,0 +1,55 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; + +import java.util.Set; + +public interface FilterableConfig extends Config { + + public abstract Config startsWith(Set filters); +} diff --git a/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java new file mode 100644 index 0000000..9e1f9da --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/FilterableConfigImpl.java @@ -0,0 +1,362 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.*; + +import java.time.Duration; +import java.time.Period; +import java.time.temporal.TemporalAmount; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class FilterableConfigImpl implements FilterableConfig { + + private final Config config; + + public FilterableConfigImpl(final Config config) { + this.config = config; + } + + @Override + public ConfigObject root() { + return config.root(); + } + + @Override + public ConfigOrigin origin() { + return config.origin(); + } + + @Override + public Config withFallback(final ConfigMergeable other) { + return config.withFallback(other); + } + + @Override + public Config resolve() { + return config.resolve(); + } + + @Override + public Config resolve(final ConfigResolveOptions options) { + return config.resolve(options); + } + + @Override + public boolean isResolved() { + return config.isResolved(); + } + + @Override + public Config resolveWith(final Config source) { + return config.resolveWith(source); + } + + @Override + public Config resolveWith(final Config source, final ConfigResolveOptions options) { + return config.resolveWith(source, options); + } + + @Override + public void checkValid(final Config reference, final String ... restrictToPaths) { + config.checkValid(reference, restrictToPaths); + } + + @Override + public boolean hasPath(final String path) { + return config.hasPath(path); + } + + @Override + public boolean hasPathOrNull(final String path) { + return config.hasPathOrNull(path); + } + + @Override + public boolean isEmpty() { + return config.isEmpty(); + } + + @Override + public Set> entrySet() { + return config.entrySet(); + } + + @Override + public boolean getIsNull(final String path) { + return config.getIsNull(path); + } + + @Override + public boolean getBoolean(final String path) { + return config.getBoolean(path); + } + + @Override + public Number getNumber(final String path) { + return config.getNumber(path); + } + + @Override + public int getInt(final String path) { + return config.getInt(path); + } + + @Override + public long getLong(final String path) { + return config.getLong(path); + } + + @Override + public double getDouble(final String path) { + return config.getDouble(path); + } + + @Override + public String getString(final String path) { + return config.getString(path); + } + + @Override + public > T getEnum(final Class enumClass, final String path) { + return config.getEnum(enumClass, path); + } + + @Override + public ConfigObject getObject(final String path) { + return config.getObject(path); + } + + @Override + public Config getConfig(final String path) { + return config.getConfig(path); + } + + @Override + public Object getAnyRef(final String path) { + return config.getAnyRef(path); + } + + @Override + public ConfigValue getValue(final String path) { + return config.getValue(path); + } + + @Override + public Long getBytes(final String path) { + return config.getBytes(path); + } + + @Override + public ConfigMemorySize getMemorySize(final String path) { + return config.getMemorySize(path); + } + + @Deprecated + @Override + public Long getMilliseconds(final String path) { + return config.getMilliseconds(path); + } + + @Deprecated + @Override + public Long getNanoseconds(final String path) { + return config.getNanoseconds(path); + } + + @Override + public long getDuration(final String path, final TimeUnit unit) { + return config.getDuration(path, unit); + } + + @Override + public Duration getDuration(final String path) { + return config.getDuration(path); + } + + @Override + public Period getPeriod(final String path) { + return config.getPeriod(path); + } + + @Override + public TemporalAmount getTemporal(final String path) { + return config.getTemporal(path); + } + + @Override + public ConfigList getList(final String path) { + return config.getList(path); + } + + @Override + public List getBooleanList(final String path) { + return config.getBooleanList(path); + } + + @Override + public List getNumberList(final String path) { + return config.getNumberList(path); + } + + @Override + public List getIntList(final String path) { + return config.getIntList(path); + } + + @Override + public List getLongList(final String path) { + return config.getLongList(path); + } + + @Override + public List getDoubleList(final String path) { + return config.getDoubleList(path); + } + + @Override + public List getStringList(final String path) { + return config.getStringList(path); + } + + @Override + public > List getEnumList(final Class enumClass, final String path) { + return config.getEnumList(enumClass, path); + } + + @Override + public List getObjectList(final String path) { + return config.getObjectList(path); + } + + @Override + public List getConfigList(final String path) { + return config.getConfigList(path); + } + + @Override + public List getAnyRefList(final String path) { + return config.getAnyRefList(path); + } + + @Override + public List getBytesList(final String path) { + return config.getBytesList(path); + } + + @Override + public List getMemorySizeList(final String path) { + return config.getMemorySizeList(path); + } + + @Deprecated + @Override + public List getMillisecondsList(final String path) { + return config.getMillisecondsList(path); + } + + @Deprecated + @Override + public List getNanosecondsList(final String path) { + return config.getNanosecondsList(path); + } + + @Override + public List getDurationList(final String path, final TimeUnit unit) { + return config.getDurationList(path, unit); + } + + @Override + public List getDurationList(final String path) { + return config.getDurationList(path); + } + + @Override + public Config withOnlyPath(final String path) { + return config.withOnlyPath(path); + } + + @Override + public Config withoutPath(final String path) { + return config.withoutPath(path); + } + + @Override + public Config atPath(final String path) { + return config.atPath(path); + } + + @Override + public Config atKey(final String key) { + return config.atKey(key); + } + + @Override + public Config withValue(final String path, final ConfigValue value) { + return config.withValue(path, value); + } + + @Override + public Config startsWith(final Set filters) { + Config filteredConfig = ConfigFactory.empty(); + + for (Map.Entry entry : config.entrySet()) { + final String key = entry.getKey(); + + for (final String filter : filters) { + if (key.startsWith(filter)) { + filteredConfig = filteredConfig.withValue(key, entry.getValue()); + } + } + } + return filteredConfig; + + } + +} diff --git a/src/main/java/com/teragrep/pth_16/InterpreterProperties.java b/src/main/java/com/teragrep/pth_16/InterpreterProperties.java new file mode 100644 index 0000000..20d7477 --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/InterpreterProperties.java @@ -0,0 +1,112 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValueFactory; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + +import java.util.Map; + +public class InterpreterProperties { + + private final JsonObject propertiesJson; + + public InterpreterProperties(final JsonObject propertiesJson) { + this.propertiesJson = propertiesJson; + } + + Config config() { + Config config = ConfigFactory.empty(); + for (Map.Entry entry : propertiesJson.entrySet()) { + final String key = entry.getKey(); + + final JsonValue entryValue = entry.getValue(); + + if (!entryValue.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException("json key <[" + key + "]> does not refer to a json object"); + } + + final JsonObject entryValueObject = entryValue.asJsonObject(); + + final String valueKey = "value"; + if (!entryValueObject.containsKey(valueKey)) { + throw new IllegalArgumentException( + "json key <[" + key + "]> does not refer to a json object with a key named <" + valueKey + ">" + ); + } + + final JsonValue jsonValue = entryValueObject.get(valueKey); + + switch (jsonValue.getValueType()) { + case STRING: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getString("value"))); + break; + case NULL: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(null)); + break; + case TRUE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(true)); + break; + case FALSE: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(false)); + break; + case NUMBER: + config = config.withValue(key, ConfigValueFactory.fromAnyRef(entryValueObject.getInt("value"))); + break; + default: + throw new IllegalArgumentException( + "json key <[" + key + "]> has object with key <" + valueKey + + "> that has unsupported value type" + jsonValue.getValueType() + ); + } + } + return config; + } + +} diff --git a/src/main/java/com/teragrep/pth_16/InterpreterSettings.java b/src/main/java/com/teragrep/pth_16/InterpreterSettings.java new file mode 100644 index 0000000..d49a4ac --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/InterpreterSettings.java @@ -0,0 +1,120 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import java.io.File; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonReader; +import jakarta.json.JsonValue; + +import java.io.FileReader; +import java.io.FileNotFoundException; + +public final class InterpreterSettings { + + private final File configFile; + + public InterpreterSettings(final File configFile) { + this.configFile = configFile; + } + + public InterpreterProperties interpreterProperties(final String interpreterName) throws FileNotFoundException { + + try (JsonReader reader = Json.createReader(new FileReader(configFile))) { + final JsonObject jsonObject = reader.readObject(); + + if (!jsonObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException("config does not contain a JSON object"); + } + + final String interpreterSettingsKey = "interpreterSettings"; + if (!jsonObject.containsKey(interpreterSettingsKey)) { + throw new IllegalArgumentException( + "config does not contain a JSON object with key <" + interpreterSettingsKey + ">" + ); + } + + final JsonObject interpreterSettings = jsonObject.getJsonObject(interpreterSettingsKey); + + if (!interpreterSettings.containsKey(interpreterName)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + + ">" + ); + } + + final JsonObject interpreterObject = interpreterSettings.getJsonObject(interpreterName); + + if (!interpreterObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + + "> as a JSON object" + ); + } + + final String propertieskey = "properties"; + if (!interpreterObject.containsKey(propertieskey)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + + "> with key <" + propertieskey + ">" + ); + } + + final JsonObject propertiesObject = interpreterObject.getJsonObject(propertieskey); + + if (!propertiesObject.getValueType().equals(JsonValue.ValueType.OBJECT)) { + throw new IllegalArgumentException( + "config does not contain a <" + interpreterSettingsKey + "> object with key <" + interpreterName + + "> with key <" + propertieskey + "> as a JSON object" + ); + } + + return new InterpreterProperties(propertiesObject); + } + } +} diff --git a/src/main/java/com/teragrep/pth_16/Query.java b/src/main/java/com/teragrep/pth_16/Query.java new file mode 100644 index 0000000..b0e5ddd --- /dev/null +++ b/src/main/java/com/teragrep/pth_16/Query.java @@ -0,0 +1,135 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.teragrep.pth_15.DPLExecutor; +import com.teragrep.pth_15.DPLExecutorFactory; +import com.teragrep.pth_15.DPLExecutorResult; +import com.typesafe.config.Config; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + +public class Query { + + private final Logger LOGGER = LoggerFactory.getLogger(Query.class); + + private final SparkSession sparkSession; + private final Config dplConfig; + private final String queryId; + private final String noteId; + private final String paragraphId; + private final String prompt; + + public Query(final SparkSession sparkSession, final Config dplConfig, final String prompt) { + this( + sparkSession, + dplConfig, + UUID.randomUUID().toString(), + "pth_16-notebook-" + UUID.randomUUID(), + "pth_16-paragraph-" + UUID.randomUUID(), + prompt + ); + } + + public Query( + final SparkSession sparkSession, + final Config dplConfig, + final String queryId, + final String noteId, + final String paragraphId, + final String prompt + ) { + this.sparkSession = sparkSession; + this.dplConfig = dplConfig; + this.queryId = queryId; + this.noteId = noteId; + this.paragraphId = paragraphId; + this.prompt = prompt; + } + + public List run() { + final AtomicReference> rows = new AtomicReference<>(new ArrayList<>()); + + final BiConsumer, Boolean> batchHandler = (rowDataset, aggsUsed) -> { + rows.set(rowDataset.toJSON().collectAsList()); + }; + + final DPLExecutor dplExecutor; + try { + dplExecutor = new DPLExecutorFactory("com.teragrep.pth_10.executor.DPLExecutorImpl", dplConfig).create(); + } + catch ( + ClassNotFoundException | NoSuchMethodException | InvocationTargetException | InstantiationException + | IllegalAccessException e + ) { + throw new RuntimeException("Error initializing DPLExecutor implementation", e); + } + + try { + final DPLExecutorResult executorResult = dplExecutor + .interpret(batchHandler, sparkSession, queryId, noteId, paragraphId, prompt); + + LOGGER.info("executorResult code <{}> message <{}>", executorResult.code(), executorResult.message()); + + } + catch (TimeoutException e) { + throw new RuntimeException(e); + } + + return rows.get(); + } +} diff --git a/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java new file mode 100644 index 0000000..e8b80df --- /dev/null +++ b/src/test/java/com/teragrep/pth_16/InterpreterSettingsConfigTest.java @@ -0,0 +1,82 @@ +/* + * Teragrep DPL Command-Line Executor (pth_16) + * Copyright (C) 2026 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ +package com.teragrep.pth_16; + +import com.typesafe.config.Config; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.HashSet; +import java.util.Set; + +public class InterpreterSettingsConfigTest { + + @Test + public void loadNFilterZep01InterpreterSettings() { + InterpreterSettings interpreterSettings = new InterpreterSettings( + new File("src/test/resources/interpreter.json") + ); + + Assertions.assertDoesNotThrow(() -> { + InterpreterProperties sparkProperties = interpreterSettings.interpreterProperties("spark"); + Config sparkConfig = sparkProperties.config(); + FilterableConfig filterableSparkConfig = new FilterableConfigImpl(sparkConfig); + + Set dplConfigFilters = new HashSet<>(); + dplConfigFilters.add("dpl."); + dplConfigFilters.add("fs.s3a."); + Config config = filterableSparkConfig.startsWith(dplConfigFilters); + + Assertions.assertFalse(config.getBoolean("dpl.pth_06.sql.log.enabled")); + + Assertions.assertEquals("putUserHere", config.getString("fs.s3a.access.key")); + + Assertions.assertFalse(config.hasPath("spark.app.name")); + }); + } + +} diff --git a/src/test/resources/interpreter.json b/src/test/resources/interpreter.json new file mode 100644 index 0000000..0d6a248 --- /dev/null +++ b/src/test/resources/interpreter.json @@ -0,0 +1,85 @@ +{ + "interpreterSettings": { + "spark": { + "id": "spark", + "name": "spark", + "group": "spark", + "properties": { + "spark.app.name": { + "name": "spark.app.name", + "value": "pth_16", + "type": "string", + "description": "The name of spark application." + }, + "fs.s3a.access.key": { + "name": "fs.s3a.access.key", + "value": "putUserHere", + "type": "string", + "description": "S3 access key" + }, + "dpl.pth_06.sql.log.enabled": { + "name": "dpl.pth_06.sql.log.enabled", + "value": false, + "type": "checkbox" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "com.teragrep.zep_01.spark.SparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sql", + "class": "com.teragrep.zep_01.spark.SparkSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "pyspark", + "class": "com.teragrep.zep_01.spark.PySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "dpl", + "class": "com.teragrep.pth_07.DPLInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "dpl", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + } + ], + "option": { + "remote": true, + "port": -1, + "perNote": "isolated", + "perUser": "isolated", + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": true + } + } + } +}