Skip to content
This repository has been archived by the owner on Nov 23, 2023. It is now read-only.

Commit

Permalink
Merge pull request #9 from 3D-e-Chem/knime-43
Browse files Browse the repository at this point in the history 
Support KNIME 4.3
  • Loading branch information
@sverhoeven
sverhoeven authored Feb 12, 2021
2 parents b30b261 + 3146f76 commit aa268d5
Show file tree
Hide file tree
Showing 12 changed files with 156 additions and 93 deletions.
18 changes: 18 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,16 @@ The file is formatted as described on http://keepachangelog.com/.

## [Unreleased]

## [2.0.4] - 2020-01-14

### Fixed

* Support KNIME 4.3 ([#9](https://github.com/3D-e-Chem/knime-python-wrapper/pull/9) + [#10](https://github.com/3D-e-Chem/knime-python-wrapper/pull/10))

### Removed

* Support for KNIME versions older than 4.3

## [2.0.3] - 2019-07-02

### Changed
Expand Down Expand Up @@ -53,3 +63,11 @@ The file is formatted as described on http://keepachangelog.com/.

* Abstract PythonWrapperNode classes
* Test utility to run tests which call PythonKernel execute

[Unreleased]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.4...HEAD
[2.0.4]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.3...v2.0.4
[2.0.3]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.2...v2.0.3
[2.0.2]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.1...v2.0.2
[2.0.1]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.1.0...v2.0.1
[1.1.0]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.0.0...v1.1.0
[1.0.0]: https://github.com/3D-e-Chem/knime-python-wrapper/releases/tag/v1.0.0
2 changes: 1 addition & 1 deletion p2/category.xml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<bundle id="nl.esciencecenter.e3dchem.python.plugin" version="2.0.3.qualifier">
<bundle id="nl.esciencecenter.e3dchem.python.plugin" version="2.0.4.qualifier">
<category name="nl.esciencecenter.3D-e-Chem"/>
</bundle>
<category-def name="nl.esciencecenter.3D-e-Chem" label="KNIME 3D-e-Chem Contributions">
Expand Down
2 changes: 1 addition & 1 deletion p2/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>nl.esciencecenter.e3dchem.python</artifactId>
<groupId>nl.esciencecenter.e3dchem.python</groupId>
<version>2.0.3-SNAPSHOT</version>
<version>2.0.4-SNAPSHOT</version>
</parent>
<artifactId>nl.esciencecenter.e3dchem.python.p2</artifactId>
<packaging>eclipse-repository</packaging>
Expand Down
16 changes: 8 additions & 8 deletions plugin/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ Manifest-Version: 1.0
Bundle-ManifestVersion: 2
Bundle-Name: Abstract Python wrapper KNIME node and helpers
Bundle-SymbolicName: nl.esciencecenter.e3dchem.python.plugin;singleton:=true
Bundle-Version: 2.0.3.qualifier
Bundle-Version: 2.0.4.qualifier
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Bundle-Vendor: Netherlands eScience Center
Require-Bundle: org.knime.core;bundle-version="[4.0.0,5.0.0)",
org.knime.base;bundle-version="[4.0.0,5.0.0)",
org.knime.python2;bundle-version="[4.0.0,5.0.0)",
org.knime.python2.serde.csv;bundle-version="[4.0.0,5.0.0)",
org.knime.python2.serde.flatbuffers;bundle-version="[4.0.0,5.0.0)",
org.knime.python2.serde.arrow;bundle-version="[4.0.0,5.0.0)",
org.knime.python2.serde.arrow.libs;bundle-version="[4.0.0,5.0.0)"
Require-Bundle: org.knime.core;bundle-version="[4.3.0,5.0.0)",
org.knime.base;bundle-version="[4.3.0,5.0.0)",
org.knime.python2;bundle-version="[4.3.0,5.0.0)",
org.knime.python2.envconfigs;bundle-version="[4.3.0,5.0.0)",
org.knime.python2.serde.csv;bundle-version="[4.3.0,5.0.0)",
org.knime.python2.serde.flatbuffers;bundle-version="[4.3.0,5.0.0)",
org.knime.python2.serde.arrow;bundle-version="[4.3.0,5.0.0)"
Bundle-ClassPath: .
Export-Package: nl.esciencecenter.e3dchem.python
2 changes: 1 addition & 1 deletion plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<groupId>nl.esciencecenter.e3dchem.python</groupId>
<artifactId>nl.esciencecenter.e3dchem.python</artifactId>
<version>2.0.3-SNAPSHOT</version>
<version>2.0.4-SNAPSHOT</version>
</parent>
<artifactId>nl.esciencecenter.e3dchem.python.plugin</artifactId>
<packaging>eclipse-plugin</packaging>
Expand Down
98 changes: 40 additions & 58 deletions plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,22 @@
package nl.esciencecenter.e3dchem.python;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;

import org.knime.python2.DefaultPythonCommand;
import org.knime.core.node.InvalidSettingsException;
import org.knime.core.node.NodeSettingsRO;
import org.knime.core.node.NodeSettingsWO;
import org.knime.core.node.workflow.FlowVariable;
import org.knime.python2.PythonCommand;
import org.knime.python2.PythonModuleSpec;
import org.knime.python2.PythonVersion;
import org.knime.python2.config.PythonCommandFlowVariableConfig;
import org.knime.python2.extensions.serializationlibrary.SentinelOption;
import org.knime.python2.extensions.serializationlibrary.SerializationOptions;
import org.knime.python2.generic.VariableNames;
import org.knime.python2.kernel.PythonKernelOptions;
import org.knime.python2.prefs.PythonPreferences;
import org.knime.core.node.InvalidSettingsException;
import org.knime.core.node.NodeSettingsRO;
import org.knime.core.node.NodeSettingsWO;
import org.knime.core.node.workflow.FlowVariable;

/**
* Configuration for {@link PythonWrapperNodeModel}.
Expand All @@ -31,32 +27,22 @@
*/
public class PythonWrapperNodeConfig {
private static final String CFG_PYTHON_VERSION_OPTION = "pythonVersionOption";
static final String CFG_PYTHON2COMMAND = "python2Command";
static final String CFG_PYTHON3COMMAND = "python3Command";
private static final String CFG_PYTHON2_COMMAND = "python2Command";

private static final String CFG_PYTHON3_COMMAND = "python3Command";
private static final String CFG_CONVERT_MISSING_TO_PYTHON = "convertMissingToPython";
private static final String CFG_CONVERT_MISSING_FROM_PYTHON = "convertMissingFromPython";
private static final String CFG_SENTINEL_OPTION = "sentinelOption";
private static final String CFG_SENTINEL_VALUE = "sentinelValue";
private static final String CFG_CHUNK_SIZE = "chunkSize";

/**
* {@link #m_python2Command} and {@link #m_python3Command} are special in that they currently aren't configurable
* via a Python scripting node's dialog but only using flow variables. If no respective flow variables are set,
* their value is retrieved from the Python preferences.
*/
private static final String INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE = "";

private PythonVersion m_pythonVersion = PythonPreferences.getPythonVersionPreference();

/**
* {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}.
*/
private PythonCommand m_python2Command = null;
private PythonCommandFlowVariableConfig m_python2CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON2_COMMAND,
PythonVersion.PYTHON2, PythonPreferences::getCondaInstallationPath);

/**
* {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}.
*/
private PythonCommand m_python3Command = null;
private PythonCommandFlowVariableConfig m_python3CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON3_COMMAND,
PythonVersion.PYTHON3, PythonPreferences::getCondaInstallationPath);

private int m_chunkSize = SerializationOptions.DEFAULT_CHUNK_SIZE;

Expand Down Expand Up @@ -118,13 +104,20 @@ public void setPythonVersion(final PythonVersion pythonVersion) {
m_pythonVersion = pythonVersion;
}

/**
* @return The config of the Python 2 command.
*/
public PythonCommandFlowVariableConfig getPython2CommandConfig() {
return m_python2CommandConfig;
}

/**
* @return The Python 2 command to use. May be {@code null} in which case no specific Python 2 command is configured
* and one has to resort to - e.g., - the {@link PythonPreferences#getPython2CommandPreference() global
* preferences}.
*/
public PythonCommand getPython2Command() {
return m_python2Command;
return m_python2CommandConfig.getCommand().orElse(null);
}

/**
Expand All @@ -133,16 +126,23 @@ public PythonCommand getPython2Command() {
* {@link PythonPreferences#getPython2CommandPreference() global preferences}.
*/
public void setPython2Command(final PythonCommand python2Command) {
m_python2Command = python2Command;
m_python2CommandConfig.setCommand(python2Command);
}

/**
* @return The config of the Python 3 command.
*/
public PythonCommandFlowVariableConfig getPython3CommandConfig() {
return m_python3CommandConfig;
}

/**
* @return The Python 3 command to use. May be {@code null} in which case no specific Python 3 command is configured
* and one has to resort to - e.g., - the {@link PythonPreferences#getPython3CommandPreference() global
* preferences}.
*/
public PythonCommand getPython3Command() {
return m_python3Command;
public PythonCommand getPython3Command() {
return m_python3CommandConfig.getCommand().orElse(null);
}

/**
Expand All @@ -151,9 +151,9 @@ public PythonCommand getPython3Command() {
* {@link PythonPreferences#getPython3CommandPreference() global preferences}.
*/
public void setPython3Command(final PythonCommand python3Command) {
m_python3Command = python3Command;
m_python3CommandConfig.setCommand(python3Command);
}

/**
*
* @return The configured number of rows to transfer to/from Python per chunk of an input/output table.
Expand Down Expand Up @@ -273,13 +273,13 @@ public void saveTo(final NodeSettingsWO settings) {
*/
public void saveToInDialog(NodeSettingsWO settings) {
settings.addString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId());
settings.addString(CFG_PYTHON2COMMAND, commandToString(getPython2Command()));
settings.addString(CFG_PYTHON3COMMAND, commandToString(getPython3Command()));
settings.addInt(CFG_CHUNK_SIZE, getChunkSize());
settings.addBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython());
settings.addBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython());
settings.addString(CFG_SENTINEL_OPTION, getSentinelOption().name());
settings.addInt(CFG_SENTINEL_VALUE, getSentinelValue());
m_python2CommandConfig.saveSettingsTo(settings);
m_python3CommandConfig.saveSettingsTo(settings);
}

/**
Expand All @@ -300,39 +300,22 @@ public void loadFrom(final NodeSettingsRO settings) throws InvalidSettingsExcept
*
* @param settings
* The settings to load from
* @throws InvalidSettingsException
*/
public void loadFromInDialog(final NodeSettingsRO settings) {
public void loadFromInDialog(final NodeSettingsRO settings) throws InvalidSettingsException {
final String pythonVersionString = settings.getString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId());
// Backward compatibility: old saved versions may be all upper case.
setPythonVersion(PythonVersion.fromId(pythonVersionString.toLowerCase(Locale.ROOT)));
final String python2CommandString =
settings.getString(CFG_PYTHON2COMMAND, commandToString(getPython2Command()));
setPython2Command(commandFromString(python2CommandString));
final String python3CommandString =
settings.getString(CFG_PYTHON3COMMAND, commandToString(getPython3Command()));
setPython3Command(commandFromString(python3CommandString));
setChunkSize(settings.getInt(CFG_CHUNK_SIZE, getChunkSize()));
setConvertMissingToPython(settings.getBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython()));
setConvertMissingFromPython(
settings.getBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython()));
setSentinelOption(SentinelOption.valueOf(settings.getString(CFG_SENTINEL_OPTION, getSentinelOption().name())));
setSentinelValue(settings.getInt(CFG_SENTINEL_VALUE, getSentinelValue()));
m_python2CommandConfig.loadSettingsFrom(settings);
m_python3CommandConfig.loadSettingsFrom(settings);
}

private static String commandToString(final PythonCommand command) {
return command != null //
? command.toString() //
: INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE;
}

private static PythonCommand commandFromString(final String commandString) {
return Objects.equals(commandString, INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE) //
? null //
// TODO: This only works for ordinary paths ("manual configuration"), not for Conda directory + environment
// name ("Conda configuration").
: new DefaultPythonCommand(commandString);
}

/**
* Set of key/value pairs which inside Python script will be a dictionary
* named by {@link #getOptionsName()}.
Expand All @@ -352,9 +335,8 @@ public Set<FlowVariable> getOptionsValues() {
public PythonKernelOptions getKernelOptions() {
final SerializationOptions serializationOptions = new SerializationOptions(getChunkSize(),
getConvertMissingToPython(), getConvertMissingFromPython(), getSentinelOption(), getSentinelValue());
PythonKernelOptions opt = new PythonKernelOptions(getPythonVersion(), getPython2Command(), getPython3Command(),
serializationOptions);
return opt.forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath);
PythonKernelOptions opt = new PythonKernelOptions();
return opt.forSerializationOptions(serializationOptions).forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath);
}

/**
Expand Down
70 changes: 58 additions & 12 deletions plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import org.knime.base.node.util.exttool.ExtToolOutputNodeModel;
Expand All @@ -22,14 +24,21 @@
import org.knime.core.node.port.PortType;
import org.knime.core.node.workflow.FlowVariable;
import org.knime.core.node.workflow.FlowVariable.Type;
import org.knime.python2.PythonCommand;
import org.knime.python2.PythonModuleSpec;
import org.knime.python2.config.PythonFlowVariableOptions;
import org.knime.python2.kernel.PythonCancelable;
import org.knime.python2.kernel.PythonCanceledExecutionException;
import org.knime.python2.kernel.PythonExecutionMonitorCancelable;
import org.knime.python2.kernel.PythonIOException;
import org.knime.python2.kernel.PythonKernel;
import org.knime.python2.kernel.PythonKernelOptions;
import org.knime.python2.kernel.PythonKernelQueue;

/**
* Implements a {@link NodeModel} for nodes that launch external Python script.
*
* @param <C>
* Configuration
* @param <C> Configuration
*/
public abstract class PythonWrapperNodeModel<C extends PythonWrapperNodeConfig> extends ExtToolOutputNodeModel {
protected C m_config = createConfig();
Expand All @@ -48,14 +57,52 @@ protected final C getConfig() {
return m_config;
}

/**
* Gets the kernel specific options.
*
* @return the kernel specific options
*/
protected PythonKernelOptions getKernelOptions() {
final PythonKernelOptions options = getConfig().getKernelOptions();
final String serializerId = new PythonFlowVariableOptions(getAvailableFlowVariables()).getSerializerId()
.orElse(null);
return options.forSerializationOptions(options.getSerializationOptions().forSerializerId(serializerId));
}

// Below has been copied from KNIME Python node code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/PythonNodeModel.java) and
// adjusted.
protected PythonKernel getNextKernelFromQueue(final PythonCancelable cancelable)
throws PythonCanceledExecutionException, PythonIOException {
return getNextKernelFromQueue(Collections.emptySet(), Collections.emptySet(), cancelable);
}

protected PythonKernel getNextKernelFromQueue(final Set<PythonModuleSpec> requiredAdditionalModules,
final PythonCancelable cancelable) throws PythonCanceledExecutionException, PythonIOException {
return getNextKernelFromQueue(requiredAdditionalModules, Collections.emptySet(), cancelable);
}

protected PythonKernel getNextKernelFromQueue(final Set<PythonModuleSpec> requiredAdditionalModules,
final Set<PythonModuleSpec> optionalAdditionalModules, final PythonCancelable cancelable)
throws PythonCanceledExecutionException, PythonIOException {
final PythonKernelOptions options = getKernelOptions();
final PythonCommand command = options.getUsePython3() //
? options.getPython3Command() //
: options.getPython2Command();
return PythonKernelQueue.getNextKernel(command, requiredAdditionalModules, optionalAdditionalModules, options,
cancelable);
}

public BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception {
// Below has been copied from Knime Python node source code and
// Below has been copied from KNIME Python node source code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/source/PythonSourceNodeModel.java) and
// adjusted.
PythonKernel kernel = new PythonKernel(getConfig().getKernelOptions());
try {
return executeKernel(inData, exec, kernel);
} finally {
kernel.close();
final PythonExecutionMonitorCancelable cancelable = new PythonExecutionMonitorCancelable(exec);
try (final PythonKernel kernel = getNextKernelFromQueue(cancelable)) {
kernel.setOptions(getConfig().getKernelOptions());
try {
return executeKernel(inData, exec, kernel);
} finally {
kernel.close();
}
}
}

Expand Down Expand Up @@ -96,11 +143,10 @@ public BufferedDataTable[] executeKernel(BufferedDataTable[] inData, ExecutionCo
/**
* Push new variables to the stack.
*
* Only pushes new variables to the stack if they are new or changed in type
* or value.
* Only pushes new variables to the stack if they are new or changed in type or
* value.
*
* @param newVariables
* The flow variables to push
* @param newVariables The flow variables to push
*/
protected void addNewVariables(Collection<FlowVariable> newVariables) {
// Below has been copied from Knime Python node source code and
Expand Down
Loading

0 comments on commit aa268d5

Please sign in to comment.