<";
+
+ /**
+ * Construct a new database connections. Note that not all these parameters are not always mandatory.
+ *
+ * @param name The database name
+ * @param type The type of database
+ * @param access The type of database access
+ * @param host The hostname or IP address
+ * @param db The database name
+ * @param port The port on which the database listens.
+ * @param user The username
+ * @param pass The password
+ */
+ public DatabaseMeta(String name, String type, String access, String host, String db, String port, String user,
+ String pass) {
+ setValues(name, type, access, host, db, port, user, pass);
+ addOptions();
+ }
+
+ /**
+ * Create an empty database connection
+ */
+ public DatabaseMeta() {
+ setDefault();
+ addOptions();
+ }
+
+ /**
+ * Set default values for an Oracle database.
+ */
+ public void setDefault() {
+ setValues("", "Oracle", "Native", "", "", "1521", "", "");
+ }
+
+ /**
+ * Add a list of common options for some databases.
+ */
+ public void addOptions() {
+ PluginInterface mySqlPlugin = PluginRegistry.getInstance().getPlugin(DatabasePluginType.class, "MYSQL");
+ PluginInterface infoBrightPlugin =
+ PluginRegistry.getInstance().getPlugin(DatabasePluginType.class, new InfobrightDatabaseMeta());
+
+ String mySQL = mySqlPlugin.getIds()[0];
+
+ addExtraOption(mySQL, "defaultFetchSize", "500");
+ addExtraOption(mySQL, "useCursorFetch", "true");
+
+ String infoBright = infoBrightPlugin.getIds()[0];
+
+ addExtraOption(infoBright, "characterEncoding", "UTF-8");
+
+ // Modern databases support this, try it by default...
+ //
+ setSupportsBooleanDataType(true);
+ setSupportsTimestampDataType(true);
+ }
+
+ /**
+ * @return the system dependend database interface for this database metadata definition
+ */
+ public DatabaseInterface getDatabaseInterface() {
+ return databaseInterface;
+ }
+
+ /**
+ * Set the system dependend database interface for this database metadata definition
+ *
+ * @param databaseInterface the system dependend database interface
+ */
+ public void setDatabaseInterface(DatabaseInterface databaseInterface) {
+ this.databaseInterface = databaseInterface;
+ }
+
+ /**
+ * Search for the right type of DatabaseInterface object and clone it.
+ *
+ * @param databaseType the type of DatabaseInterface to look for (description)
+ * @return The requested DatabaseInterface
+ * @throws KettleDatabaseException when the type could not be found or referenced.
+ */
+ public static final DatabaseInterface getDatabaseInterface(String databaseType) throws KettleDatabaseException {
+ DatabaseInterface di = findDatabaseInterface(databaseType);
+ if (di == null) {
+ throw new KettleDatabaseException(BaseMessages.getString(
+ PKG, "DatabaseMeta.Error.DatabaseInterfaceNotFound", databaseType));
+ }
+ return (DatabaseInterface) di.clone();
+ }
+
+ /**
+ * Search for the right type of DatabaseInterface object and return it.
+ *
+ * @param databaseTypeDesc the type of DatabaseInterface to look for (id or description)
+ * @return The requested DatabaseInterface
+ * @throws KettleDatabaseException when the type could not be found or referenced.
+ */
+ private static final DatabaseInterface findDatabaseInterface(String databaseTypeDesc) throws KettleDatabaseException {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ PluginInterface plugin = registry.getPlugin(DatabasePluginType.class, databaseTypeDesc);
+ if (plugin == null) {
+ plugin = registry.findPluginWithName(DatabasePluginType.class, databaseTypeDesc);
+ }
+
+ if (plugin == null) {
+ throw new KettleDatabaseException("database type with plugin id ["
+ + databaseTypeDesc + "] couldn't be found!");
+ }
+
+ return getDatabaseInterfacesMap().get(plugin.getIds()[0]);
+ }
+
+ /**
+ * Returns the database ID of this database connection if a repository was used before.
+ *
+ * @return the ID of the db connection.
+ */
+ @Override
+ public ObjectId getObjectId() {
+ return databaseInterface.getObjectId();
+ }
+
+ @Override
+ public void setObjectId(ObjectId id) {
+ databaseInterface.setObjectId(id);
+ }
+
+ @Override
+ public Object clone() {
+ DatabaseMeta databaseMeta = new DatabaseMeta();
+ databaseMeta.replaceMeta(this);
+ databaseMeta.setObjectId(null);
+ return databaseMeta;
+ }
+
+ public void replaceMeta(DatabaseMeta databaseMeta) {
+ this.setValues(
+ databaseMeta.getName(), databaseMeta.getPluginId(), databaseMeta.getAccessTypeDesc(), databaseMeta
+ .getHostname(), databaseMeta.getDatabaseName(), databaseMeta.getDatabasePortNumberString(),
+ databaseMeta.getUsername(), databaseMeta.getPassword());
+ this.setServername(databaseMeta.getServername());
+ this.setDataTablespace(databaseMeta.getDataTablespace());
+ this.setIndexTablespace(databaseMeta.getIndexTablespace());
+
+ this.databaseInterface = (DatabaseInterface) databaseMeta.databaseInterface.clone();
+
+ this.setObjectId(databaseMeta.getObjectId());
+ this.setChanged();
+ }
+
+ public void setValues(String name, String type, String access, String host, String db, String port,
+ String user, String pass) {
+ try {
+ databaseInterface = getDatabaseInterface(type);
+ } catch (KettleDatabaseException kde) {
+ throw new RuntimeException("Database type not found!", kde);
+ }
+
+ setName(name);
+ setAccessType(getAccessType(access));
+ setHostname(host);
+ setDBName(db);
+ setDBPort(port);
+ setUsername(user);
+ setPassword(pass);
+ setServername(null);
+ setChanged(false);
+ }
+
+ public void setDatabaseType(String type) {
+ DatabaseInterface oldInterface = databaseInterface;
+
+ try {
+ databaseInterface = getDatabaseInterface(type);
+ } catch (KettleDatabaseException kde) {
+ throw new RuntimeException("Database type [" + type + "] not found!", kde);
+ }
+
+ setName(oldInterface.getName());
+ setDisplayName(oldInterface.getDisplayName());
+ setAccessType(oldInterface.getAccessType());
+ setHostname(oldInterface.getHostname());
+ setDBName(oldInterface.getDatabaseName());
+ setDBPort(oldInterface.getDatabasePortNumberString());
+ setUsername(oldInterface.getUsername());
+ setPassword(oldInterface.getPassword());
+ setServername(oldInterface.getServername());
+ setDataTablespace(oldInterface.getDataTablespace());
+ setIndexTablespace(oldInterface.getIndexTablespace());
+ setChanged(oldInterface.isChanged());
+ }
+
+ public void setValues(DatabaseMeta info) {
+ databaseInterface = (DatabaseInterface) info.databaseInterface.clone();
+ }
+
+ /**
+ * Sets the name of the database connection. This name should be unique in a transformation and in general in a single
+ * repository.
+ *
+ * @param name The name of the database connection
+ */
+ @Override
+ public void setName(String name) {
+ databaseInterface.setName(name);
+ }
+
+ /**
+ * Returns the name of the database connection
+ *
+ * @return The name of the database connection
+ */
+ @Override
+ public String getName() {
+ return databaseInterface.getName();
+ }
+
+ public void setDisplayName(String displayName) {
+ databaseInterface.setDisplayName(displayName);
+ }
+
+ /**
+ * Returns the name of the database connection
+ *
+ * @return The name of the database connection
+ */
+ public String getDisplayName() {
+ return databaseInterface.getDisplayName();
+ }
+
+ /**
+ * Returns the type of database, one of
+ *
+ * TYPE_DATABASE_MYSQL
+ *
+ * TYPE_DATABASE_ORACLE
+ *
+ * TYPE_DATABASE_...
+ *
+ *
+ * @return the database type
+ * @Deprecated public int getDatabaseType() { return databaseInterface.getDatabaseType(); }
+ */
+
+ /**
+ * The plugin ID of the database interface
+ */
+ public String getPluginId() {
+ return databaseInterface.getPluginId();
+ }
+
+ /*
+ * Sets the type of database.
+ *
+ * @param db_type The database type public void setDatabaseType(int db_type) { databaseInterface this.databaseType =
+ * db_type; }
+ */
+
+ /**
+ * Return the type of database access. One of
+ *
+ * TYPE_ACCESS_NATIVE
+ *
+ * TYPE_ACCESS_ODBC
+ *
+ * TYPE_ACCESS_OCI
+ *
+ *
+ * @return The type of database access.
+ */
+ public int getAccessType() {
+ return databaseInterface.getAccessType();
+ }
+
+ /**
+ * Set the type of database access.
+ *
+ * @param access_type The access type.
+ */
+ public void setAccessType(int access_type) {
+ databaseInterface.setAccessType(access_type);
+ }
+
+ /**
+ * Returns a short description of the type of database.
+ *
+ * @return A short description of the type of database.
+ * @deprecated This is actually the plugin ID
+ */
+ @Deprecated
+ public String getDatabaseTypeDesc() {
+ return getPluginId();
+ }
+
+ /**
+ * Gets you a short description of the type of database access.
+ *
+ * @return A short description of the type of database access.
+ */
+ public String getAccessTypeDesc() {
+ return dbAccessTypeCode[getAccessType()];
+ }
+
+ /**
+ * Return the hostname of the machine on which the database runs.
+ *
+ * @return The hostname of the database.
+ */
+ public String getHostname() {
+ return databaseInterface.getHostname();
+ }
+
+ /**
+ * Sets the hostname of the machine on which the database runs.
+ *
+ * @param hostname The hostname of the machine on which the database runs.
+ */
+ public void setHostname(String hostname) {
+ databaseInterface.setHostname(hostname);
+ }
+
+ /**
+ * Return the port on which the database listens as a String. Allows for parameterisation.
+ *
+ * @return The database port.
+ */
+ public String getDatabasePortNumberString() {
+ return databaseInterface.getDatabasePortNumberString();
+ }
+
+ /**
+ * Sets the port on which the database listens.
+ *
+ * @param db_port The port number on which the database listens
+ */
+ public void setDBPort(String db_port) {
+ databaseInterface.setDatabasePortNumberString(db_port);
+ }
+
+ /**
+ * Return the name of the database.
+ *
+ * @return The database name.
+ */
+ public String getDatabaseName() {
+ return databaseInterface.getDatabaseName();
+ }
+
+ /**
+ * Set the name of the database.
+ *
+ * @param databaseName The new name of the database
+ */
+ public void setDBName(String databaseName) {
+ databaseInterface.setDatabaseName(databaseName);
+ }
+
+ /**
+ * Get the username to log into the database on this connection.
+ *
+ * @return The username to log into the database on this connection.
+ */
+ public String getUsername() {
+ return databaseInterface.getUsername();
+ }
+
+ /**
+ * Sets the username to log into the database on this connection.
+ *
+ * @param username The username
+ */
+ public void setUsername(String username) {
+ databaseInterface.setUsername(username);
+ }
+
+ /**
+ * Get the password to log into the database on this connection.
+ *
+ * @return the password to log into the database on this connection.
+ */
+ public String getPassword() {
+ return databaseInterface.getPassword();
+ }
+
+ /**
+ * Sets the password to log into the database on this connection.
+ *
+ * @param password the password to log into the database on this connection.
+ */
+ public void setPassword(String password) {
+ databaseInterface.setPassword(password);
+ }
+
+ /**
+ * @param servername the Informix servername
+ */
+ public void setServername(String servername) {
+ databaseInterface.setServername(servername);
+ }
+
+ /**
+ * @return the Informix servername
+ */
+ public String getServername() {
+ return databaseInterface.getServername();
+ }
+
+ public String getDataTablespace() {
+ return databaseInterface.getDataTablespace();
+ }
+
+ public void setDataTablespace(String data_tablespace) {
+ databaseInterface.setDataTablespace(data_tablespace);
+ }
+
+ public String getIndexTablespace() {
+ return databaseInterface.getIndexTablespace();
+ }
+
+ public void setIndexTablespace(String index_tablespace) {
+ databaseInterface.setIndexTablespace(index_tablespace);
+ }
+
+ public void setChanged() {
+ setChanged(true);
+ }
+
+ public void setChanged(boolean ch) {
+ databaseInterface.setChanged(ch);
+ }
+
+ public boolean hasChanged() {
+ return databaseInterface.isChanged();
+ }
+
+ public void clearChanged() {
+ databaseInterface.setChanged(false);
+ }
+
+ @Override
+ public String toString() {
+ return getDisplayName();
+ }
+
+ /**
+ * @return The extra attributes for this database connection
+ */
+ public Properties getAttributes() {
+ return databaseInterface.getAttributes();
+ }
+
+ /**
+ * Set extra attributes on this database connection
+ *
+ * @param attributes The extra attributes to set on this database connection.
+ */
+ public void setAttributes(Properties attributes) {
+ databaseInterface.setAttributes(attributes);
+ }
+
+ /**
+ * Constructs a new database using an XML string snippet. It expects the snippet to be enclosed in
+ * connection tags.
+ *
+ * @param xml The XML string to parse
+ * @throws KettleXMLException in case there is an XML parsing error
+ */
+ public DatabaseMeta(String xml) throws KettleXMLException {
+ this(XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), "connection"));
+ }
+
+ /**
+ * Reads the information from an XML Node into this new database connection.
+ *
+ * @param con The Node to read the data from
+ * @throws KettleXMLException
+ */
+ public DatabaseMeta(Node con) throws KettleXMLException {
+ this();
+
+ try {
+ String type = XMLHandler.getTagValue(con, "type");
+ try {
+ databaseInterface = getDatabaseInterface(type);
+
+ } catch (KettleDatabaseException kde) {
+ throw new KettleXMLException("Unable to create new database interface", kde);
+ }
+
+ setName(XMLHandler.getTagValue(con, "name"));
+ setDisplayName(getName());
+ setHostname(XMLHandler.getTagValue(con, "server"));
+ String acc = XMLHandler.getTagValue(con, "access");
+ setAccessType(getAccessType(acc));
+
+ setDBName(XMLHandler.getTagValue(con, "database"));
+
+ // The DB port is read here too for backward compatibility! getName()
+ //
+ setDBPort(XMLHandler.getTagValue(con, "port"));
+ setUsername(XMLHandler.getTagValue(con, "username"));
+ setPassword(Encr.decryptPasswordOptionallyEncrypted(XMLHandler.getTagValue(con, "password")));
+ setServername(XMLHandler.getTagValue(con, "servername"));
+ setDataTablespace(XMLHandler.getTagValue(con, "data_tablespace"));
+ setIndexTablespace(XMLHandler.getTagValue(con, "index_tablespace"));
+
+ setReadOnly(Boolean.valueOf(XMLHandler.getTagValue(con, "read_only")));
+
+ // Also, read the database attributes...
+ Node attrsnode = XMLHandler.getSubNode(con, "attributes");
+ if (attrsnode != null) {
+ List attrnodes = XMLHandler.getNodes(attrsnode, "attribute");
+ for (Node attrnode : attrnodes) {
+ String code = XMLHandler.getTagValue(attrnode, "code");
+ String attribute = XMLHandler.getTagValue(attrnode, "attribute");
+ if (code != null && attribute != null) {
+ getAttributes().put(code, attribute);
+ }
+ getDatabasePortNumberString();
+ }
+ }
+ } catch (Exception e) {
+ throw new KettleXMLException("Unable to load database connection info from XML node", e);
+ }
+ }
+
+ @Override
+ public String getXML() {
+ StringBuffer retval = new StringBuffer(250);
+
+ retval.append(" <").append(XML_TAG).append('>').append(Const.CR);
+ retval.append(" ").append(XMLHandler.addTagValue("name", getName()));
+ retval.append(" ").append(XMLHandler.addTagValue("server", getHostname()));
+ retval.append(" ").append(XMLHandler.addTagValue("type", getPluginId()));
+ retval.append(" ").append(XMLHandler.addTagValue("access", getAccessTypeDesc()));
+ retval.append(" ").append(XMLHandler.addTagValue("database", getDatabaseName()));
+ retval.append(" ").append(XMLHandler.addTagValue("port", getDatabasePortNumberString()));
+ retval.append(" ").append(XMLHandler.addTagValue("username", getUsername()));
+ retval.append(" ").append(
+ XMLHandler.addTagValue("password", Encr.encryptPasswordIfNotUsingVariables(getPassword())));
+ retval.append(" ").append(XMLHandler.addTagValue("servername", getServername()));
+ retval.append(" ").append(XMLHandler.addTagValue("data_tablespace", getDataTablespace()));
+ retval.append(" ").append(XMLHandler.addTagValue("index_tablespace", getIndexTablespace()));
+
+ // only write the tag out if it is set to true
+ if (isReadOnly()) {
+ retval.append(" ").append(XMLHandler.addTagValue("read_only", Boolean.toString(isReadOnly())));
+ }
+
+ retval.append(" ").append(Const.CR);
+
+ List list = new ArrayList();
+ Set keySet = getAttributes().keySet();
+ for (Object object : keySet) {
+ list.add((String) object);
+ }
+ Collections.sort(list); // Sort the entry-sets to make sure we can compare XML strings: if the order is different,
+ // the XML is different.
+
+ for (Iterator iter = list.iterator(); iter.hasNext(); ) {
+ String code = iter.next();
+ String attribute = getAttributes().getProperty(code);
+ if (!Const.isEmpty(attribute)) {
+ retval.append(" "
+ + XMLHandler.addTagValue("code", code, false)
+ + XMLHandler.addTagValue("attribute", attribute, false) + " " + Const.CR);
+ }
+ }
+ retval.append(" ").append(Const.CR);
+
+ retval.append(" ").append(Const.CR);
+ return retval.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode(); // name of connection is unique!
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof DatabaseMeta && getName().equals(((DatabaseMeta) obj).getName());
+ }
+
+ public String getURL() throws KettleDatabaseException {
+ return getURL(null);
+ }
+
+ public String getURL(String partitionId) throws KettleDatabaseException {
+ // First see if we're not doing any JNDI...
+ //
+ /*
+ * This doesn't make much sense here - we check but do nothing? if ( getAccessType() == TYPE_ACCESS_JNDI ) { // We
+ * can't really determine the URL here. // // }
+ */
+ String baseUrl;
+ String hostname;
+ String port;
+ String databaseName;
+
+ if (isPartitioned() && !Const.isEmpty(partitionId)) {
+ // Get the cluster information...
+ PartitionDatabaseMeta partition = getPartitionMeta(partitionId);
+ hostname = environmentSubstitute(partition.getHostname());
+ port = environmentSubstitute(partition.getPort());
+ databaseName = environmentSubstitute(partition.getDatabaseName());
+ } else {
+ hostname = environmentSubstitute(getHostname());
+ port = environmentSubstitute(getDatabasePortNumberString());
+ databaseName = environmentSubstitute(getDatabaseName());
+ }
+ baseUrl = databaseInterface.getURL(environmentSubstitute(hostname), environmentSubstitute(port),
+ environmentSubstitute(databaseName));
+ StringBuffer url = new StringBuffer(environmentSubstitute(baseUrl));
+
+ if (databaseInterface.supportsOptionsInURL()) {
+ // OK, now add all the options...
+ String optionIndicator = getExtraOptionIndicator();
+ String optionSeparator = getExtraOptionSeparator();
+ String valueSeparator = getExtraOptionValueSeparator();
+
+ Map map = getExtraOptions();
+ if (map.size() > 0) {
+ Iterator iterator = map.keySet().iterator();
+ boolean first = true;
+ while (iterator.hasNext()) {
+ String typedParameter = iterator.next();
+ int dotIndex = typedParameter.indexOf('.');
+ if (dotIndex >= 0) {
+ String typeCode = typedParameter.substring(0, dotIndex);
+ String parameter = typedParameter.substring(dotIndex + 1);
+ String value = map.get(typedParameter);
+
+ // Only add to the URL if it's the same database type code...
+ //
+ if (databaseInterface.getPluginId().equals(typeCode)) {
+ if (first && url.indexOf(valueSeparator) == -1) {
+ url.append(optionIndicator);
+ } else {
+ url.append(optionSeparator);
+ }
+
+ url.append(parameter);
+ if (!Const.isEmpty(value) && !value.equals(EMPTY_OPTIONS_STRING)) {
+ url.append(valueSeparator).append(value);
+ }
+ first = false;
+ }
+ }
+ }
+ }
+ }
+ // else {
+ // We need to put all these options in a Properties file later (Oracle & Co.)
+ // This happens at connect time...
+ // }
+
+ return url.toString();
+ }
+
+ public Properties getConnectionProperties() {
+ Properties properties = new Properties();
+
+ Map map = getExtraOptions();
+ if (map.size() > 0) {
+ Iterator iterator = map.keySet().iterator();
+ while (iterator.hasNext()) {
+ String typedParameter = iterator.next();
+ int dotIndex = typedParameter.indexOf('.');
+ if (dotIndex >= 0) {
+ String typeCode = typedParameter.substring(0, dotIndex);
+ String parameter = typedParameter.substring(dotIndex + 1);
+ String value = map.get(typedParameter);
+
+ // Only add to the URL if it's the same database type code...
+ //
+ if (databaseInterface.getPluginId().equals(typeCode)) {
+ if (value != null && value.equals(EMPTY_OPTIONS_STRING)) {
+ value = "";
+ }
+ properties.put(parameter, environmentSubstitute(Const.NVL(value, "")));
+ }
+ }
+ }
+ }
+
+ return properties;
+ }
+
+ public String getExtraOptionIndicator() {
+ return databaseInterface.getExtraOptionIndicator();
+ }
+
+ /**
+ * @return The extra option separator in database URL for this platform (usually this is semicolon ; )
+ */
+ public String getExtraOptionSeparator() {
+ return databaseInterface.getExtraOptionSeparator();
+ }
+
+ /**
+ * @return The extra option value separator in database URL for this platform (usually this is the equal sign = )
+ */
+ public String getExtraOptionValueSeparator() {
+ return databaseInterface.getExtraOptionValueSeparator();
+ }
+
+ /**
+ * Add an extra option to the attributes list
+ *
+ * @param databaseTypeCode The database type code for which the option applies
+ * @param option The option to set
+ * @param value The value of the option
+ */
+ public void addExtraOption(String databaseTypeCode, String option, String value) {
+ databaseInterface.addExtraOption(databaseTypeCode, option, value);
+ }
+
+ public void applyDefaultOptions(DatabaseInterface databaseInterface) {
+ final Map extraOptions = getExtraOptions();
+
+ final Map defaultOptions = databaseInterface.getDefaultOptions();
+ for (String option : defaultOptions.keySet()) {
+ String value = defaultOptions.get(option);
+ String[] split = option.split("[.]", 2);
+ if (!extraOptions.containsKey(option) && split.length == 2) {
+ addExtraOption(split[0], split[1], value);
+ }
+ }
+ }
+
+ /**
+ * @return true if the database supports transactions
+ * @deprecated because the same database can support transactions or not. It all depends on the database setup.
+ * Therefor, we look at the database metadata DatabaseMetaData.supportsTransactions() in stead of this.
+ */
+ @Deprecated
+ public boolean supportsTransactions() {
+ return databaseInterface.supportsTransactions();
+ }
+
+ public boolean supportsAutoinc() {
+ return databaseInterface.supportsAutoInc();
+ }
+
+ public boolean supportsSequences() {
+ return databaseInterface.supportsSequences();
+ }
+
+ public String getSQLSequenceExists(String sequenceName) {
+ return databaseInterface.getSQLSequenceExists(sequenceName);
+ }
+
+ public boolean supportsBitmapIndex() {
+ return databaseInterface.supportsBitmapIndex();
+ }
+
+ public boolean supportsSetLong() {
+ return databaseInterface.supportsSetLong();
+ }
+
+ /**
+ * @return true if the database supports schemas
+ */
+ public boolean supportsSchemas() {
+ return databaseInterface.supportsSchemas();
+ }
+
+ /**
+ * @return true if the database supports catalogs
+ */
+ public boolean supportsCatalogs() {
+ return databaseInterface.supportsCatalogs();
+ }
+
+ /**
+ * @return true when the database engine supports empty transaction. (for example Informix does not on a non-ANSI
+ * database type!)
+ */
+ public boolean supportsEmptyTransactions() {
+ return databaseInterface.supportsEmptyTransactions();
+ }
+
+ /**
+ * See if this database supports the setCharacterStream() method on a PreparedStatement.
+ *
+ * @return true if we can set a Stream on a field in a PreparedStatement. False if not.
+ */
+ public boolean supportsSetCharacterStream() {
+ return databaseInterface.supportsSetCharacterStream();
+ }
+
+ /**
+ * Get the maximum length of a text field for this database connection. This includes optional CLOB, Memo and Text
+ * fields. (the maximum!)
+ *
+ * @return The maximum text field length for this database type. (mostly CLOB_LENGTH)
+ */
+ public int getMaxTextFieldLength() {
+ return databaseInterface.getMaxTextFieldLength();
+ }
+
+ public static final int getAccessType(String dbaccess) {
+ int i;
+
+ if (dbaccess == null) {
+ return TYPE_ACCESS_NATIVE;
+ }
+
+ for (i = 0; i < dbAccessTypeCode.length; i++) {
+ if (dbAccessTypeCode[i].equalsIgnoreCase(dbaccess)) {
+ return i;
+ }
+ }
+ for (i = 0; i < dbAccessTypeDesc.length; i++) {
+ if (dbAccessTypeDesc[i].equalsIgnoreCase(dbaccess)) {
+ return i;
+ }
+ }
+
+ return TYPE_ACCESS_NATIVE;
+ }
+
+ public static final String getAccessTypeDesc(int dbaccess) {
+ if (dbaccess < 0) {
+ return null;
+ }
+ if (dbaccess > dbAccessTypeCode.length) {
+ return null;
+ }
+
+ return dbAccessTypeCode[dbaccess];
+ }
+
+ public static final String getAccessTypeDescLong(int dbaccess) {
+ if (dbaccess < 0) {
+ return null;
+ }
+ if (dbaccess > dbAccessTypeDesc.length) {
+ return null;
+ }
+
+ return dbAccessTypeDesc[dbaccess];
+ }
+
+ public static final DatabaseInterface[] getDatabaseInterfaces() {
+ List list = new ArrayList(getDatabaseInterfacesMap().values());
+ return list.toArray(new DatabaseInterface[list.size()]);
+ }
+
+ /**
+ * Clear the database interfaces map. The map is cached by getDatabaseInterfacesMap(), but in some instances it may
+ * need to be reloaded (such as adding/updating Database plugins). After calling clearDatabaseInterfacesMap(), the
+ * next call to getDatabaseInterfacesMap() will reload the map.
+ */
+ public static final void clearDatabaseInterfacesMap() {
+ allDatabaseInterfaces = null;
+ }
+
+ private static final Future> createDatabaseInterfacesMap() {
+ return ExecutorUtil.getExecutor().submit(new Callable>() {
+ private Map doCreate() {
+ LogChannelInterface log = LogChannel.GENERAL;
+ PluginRegistry registry = PluginRegistry.getInstance();
+
+ List plugins = registry.getPlugins(DatabasePluginType.class);
+ HashMap tmpAllDatabaseInterfaces = new HashMap();
+ for (PluginInterface plugin : plugins) {
+ try {
+ DatabaseInterface databaseInterface = (DatabaseInterface) registry.loadClass(plugin);
+ databaseInterface.setPluginId(plugin.getIds()[0]);
+ databaseInterface.setPluginName(plugin.getName());
+ tmpAllDatabaseInterfaces.put(plugin.getIds()[0], databaseInterface);
+ } catch (KettlePluginException cnfe) {
+ System.out.println("Could not create connection entry for "
+ + plugin.getName() + ". " + cnfe.getCause().getClass().getName());
+ log.logError("Could not create connection entry for "
+ + plugin.getName() + ". " + cnfe.getCause().getClass().getName());
+ if (log.isDebug()) {
+ log.logDebug("Debug-Error loading plugin: " + plugin, cnfe);
+ }
+ } catch (Exception e) {
+ log.logError("Error loading plugin: " + plugin, e);
+ }
+ }
+ return Collections.unmodifiableMap(tmpAllDatabaseInterfaces);
+ }
+
+ @Override
+ public Map call() throws Exception {
+ return doCreate();
+ }
+ });
+ }
+
+ public static final Map getDatabaseInterfacesMap() {
+ Future> allDatabaseInterfaces = DatabaseMeta.allDatabaseInterfaces;
+ while (allDatabaseInterfaces == null) {
+ DatabaseMeta.allDatabaseInterfaces = createDatabaseInterfacesMap();
+ allDatabaseInterfaces = DatabaseMeta.allDatabaseInterfaces;
+ }
+ try {
+ return allDatabaseInterfaces.get();
+ } catch (Exception e) {
+ clearDatabaseInterfacesMap();
+ // doCreate() above doesn't declare any exceptions so anything that comes out SHOULD be a runtime exception
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ } else {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public static final int[] getAccessTypeList(String dbTypeDesc) {
+ try {
+ DatabaseInterface di = findDatabaseInterface(dbTypeDesc);
+ return di.getAccessTypeList();
+ } catch (KettleDatabaseException kde) {
+ return null;
+ }
+ }
+
+ public static final int getPortForDBType(String strtype, String straccess) {
+ try {
+ DatabaseInterface di = getDatabaseInterface(strtype);
+ di.setAccessType(getAccessType(straccess));
+ return di.getDefaultDatabasePort();
+ } catch (KettleDatabaseException kde) {
+ return -1;
+ }
+ }
+
+ public int getDefaultDatabasePort() {
+ return databaseInterface.getDefaultDatabasePort();
+ }
+
+ public int getNotFoundTK(boolean use_autoinc) {
+ return databaseInterface.getNotFoundTK(use_autoinc);
+ }
+
+ public String getDriverClass() {
+ return environmentSubstitute(databaseInterface.getDriverClass());
+ }
+
+ public String stripCR(String sbsql) {
+ if (sbsql == null) {
+ return null;
+ }
+ return stripCR(new StringBuffer(sbsql));
+ }
+
+ public String stripCR(StringBuffer sbsql) {
+ // DB2 Can't handle \n in SQL Statements...
+ if (!supportsNewLinesInSQL()) {
+ // Remove CR's
+ for (int i = sbsql.length() - 1; i >= 0; i--) {
+ if (sbsql.charAt(i) == '\n' || sbsql.charAt(i) == '\r') {
+ sbsql.setCharAt(i, ' ');
+ }
+ }
+ }
+
+ return sbsql.toString();
+ }
+
+ public String getSeqNextvalSQL(String sequenceName) {
+ return databaseInterface.getSQLNextSequenceValue(sequenceName);
+ }
+
+ public String getSQLCurrentSequenceValue(String sequenceName) {
+ return databaseInterface.getSQLCurrentSequenceValue(sequenceName);
+ }
+
+ public boolean isFetchSizeSupported() {
+ return databaseInterface.isFetchSizeSupported();
+ }
+
+ /**
+ * Indicates the need to insert a placeholder (0) for auto increment fields.
+ *
+ * @return true if we need a placeholder for auto increment fields in insert statements.
+ */
+ public boolean needsPlaceHolder() {
+ return databaseInterface.needsPlaceHolder();
+ }
+
+ public String getFunctionSum() {
+ return databaseInterface.getFunctionSum();
+ }
+
+ public String getFunctionAverage() {
+ return databaseInterface.getFunctionAverage();
+ }
+
+ public String getFunctionMaximum() {
+ return databaseInterface.getFunctionMaximum();
+ }
+
+ public String getFunctionMinimum() {
+ return databaseInterface.getFunctionMinimum();
+ }
+
+ public String getFunctionCount() {
+ return databaseInterface.getFunctionCount();
+ }
+
+ /**
+ * Check the database connection parameters and give back an array of remarks
+ *
+ * @return an array of remarks Strings
+ */
+ public String[] checkParameters() {
+ ArrayList remarks = new ArrayList();
+
+ if (getDatabaseInterface() == null) {
+ remarks.add("No database type was choosen");
+ }
+
+ if (getName() == null || getName().length() == 0) {
+ remarks.add("Please give this database connection a name");
+ }
+
+ if (!isPartitioned()
+ && !(getDatabaseInterface() instanceof SAPR3DatabaseMeta
+ || getDatabaseInterface() instanceof GenericDatabaseMeta)) {
+ if (getDatabaseName() == null || getDatabaseName().length() == 0) {
+ remarks.add("Please specify the name of the database");
+ }
+ }
+
+ return remarks.toArray(new String[remarks.size()]);
+ }
+
+ /**
+ * This is now replaced with getQuotedSchemaTableCombination(), enforcing the use of the quoteFields call
+ *
+ * @param schemaName
+ * @param tableName
+ * @return
+ * @deprecated please use getQuotedSchemaTableCombination()
+ */
+ @Deprecated
+ public String getSchemaTableCombination(String schemaName, String tableName) {
+ return getQuotedSchemaTableCombination(schemaName, tableName);
+ }
+
+ /**
+ * Calculate the schema-table combination, usually this is the schema and table separated with a dot. (schema.table)
+ *
+ * @param schemaName the schema-name or null if no schema is used.
+ * @param tableName the table name
+ * @return the schemaname-tablename combination
+ */
+ public String getQuotedSchemaTableCombination(String schemaName, String tableName) {
+ if (Const.isEmpty(schemaName)) {
+ if (Const.isEmpty(getPreferredSchemaName())) {
+ return quoteField(environmentSubstitute(tableName)); // no need to look further
+ } else {
+ return databaseInterface.getSchemaTableCombination(
+ quoteField(environmentSubstitute(getPreferredSchemaName())),
+ quoteField(environmentSubstitute(tableName)));
+ }
+ } else {
+ return databaseInterface.getSchemaTableCombination(
+ quoteField(environmentSubstitute(schemaName)), quoteField(environmentSubstitute(tableName)));
+ }
+ }
+
+ public boolean isClob(ValueMetaInterface v) {
+ boolean retval = true;
+
+ if (v == null || v.getLength() < DatabaseMeta.CLOB_LENGTH) {
+ retval = false;
+ } else {
+ return true;
+ }
+ return retval;
+ }
+
+ public String getFieldDefinition(ValueMetaInterface v, String tk, String pk, boolean use_autoinc) {
+ return getFieldDefinition(v, tk, pk, use_autoinc, true, true);
+ }
+
+ public String getFieldDefinition(ValueMetaInterface v, String tk, String pk, boolean use_autoinc,
+ boolean add_fieldname, boolean add_cr) {
+
+ String definition =
+ v.getDatabaseColumnTypeDefinition(databaseInterface, tk, pk, use_autoinc, add_fieldname, add_cr);
+ if (!Const.isEmpty(definition)) {
+ return definition;
+ }
+
+ return databaseInterface.getFieldDefinition(v, tk, pk, use_autoinc, add_fieldname, add_cr);
+ }
+
+ public String getLimitClause(int nrRows) {
+ return databaseInterface.getLimitClause(nrRows);
+ }
+
+ /**
+ * @param tableName The table or schema-table combination. We expect this to be quoted properly already!
+ * @return the SQL for to get the fields of this table.
+ */
+ public String getSQLQueryFields(String tableName) {
+ return databaseInterface.getSQLQueryFields(tableName);
+ }
+
+ public String getAddColumnStatement(String tablename, ValueMetaInterface v, String tk, boolean use_autoinc,
+ String pk, boolean semicolon) {
+ String retval = databaseInterface.getAddColumnStatement(tablename, v, tk, use_autoinc, pk, semicolon);
+ retval += Const.CR;
+ if (semicolon) {
+ retval += ";" + Const.CR;
+ }
+ return retval;
+ }
+
+ public String getDropColumnStatement(String tablename, ValueMetaInterface v, String tk, boolean use_autoinc,
+ String pk, boolean semicolon) {
+ String retval = databaseInterface.getDropColumnStatement(tablename, v, tk, use_autoinc, pk, semicolon);
+ retval += Const.CR;
+ if (semicolon) {
+ retval += ";" + Const.CR;
+ }
+ return retval;
+ }
+
+ public String getModifyColumnStatement(String tablename, ValueMetaInterface v, String tk, boolean use_autoinc,
+ String pk, boolean semicolon) {
+ String retval = databaseInterface.getModifyColumnStatement(tablename, v, tk, use_autoinc, pk, semicolon);
+ retval += Const.CR;
+ if (semicolon) {
+ retval += ";" + Const.CR;
+ }
+
+ return retval;
+ }
+
+ /**
+ * @return an array of reserved words for the database type...
+ */
+ public String[] getReservedWords() {
+ return databaseInterface.getReservedWords();
+ }
+
+ /**
+ * @return true if reserved words need to be double quoted ("password", "select", ...)
+ */
+ public boolean quoteReservedWords() {
+ return databaseInterface.quoteReservedWords();
+ }
+
+ /**
+ * @return The start quote sequence, mostly just double quote, but sometimes [, ...
+ */
+ public String getStartQuote() {
+ return databaseInterface.getStartQuote();
+ }
+
+ /**
+ * @return The end quote sequence, mostly just double quote, but sometimes ], ...
+ */
+ public String getEndQuote() {
+ return databaseInterface.getEndQuote();
+ }
+
+ /**
+ * Returns a quoted field if this is needed: contains spaces, is a reserved word, ...
+ *
+ * @param field The fieldname to check for quoting
+ * @return The quoted field (if this is needed.
+ */
+ public String quoteField(String field) {
+ if (Const.isEmpty(field)) {
+ return null;
+ }
+
+ if (isForcingIdentifiersToLowerCase()) {
+ field = field.toLowerCase();
+ } else if (isForcingIdentifiersToUpperCase()) {
+ field = field.toUpperCase();
+ }
+
+ // If the field already contains quotes, we don't touch it anymore, just return the same string...
+ if (field.indexOf(getStartQuote()) >= 0 || field.indexOf(getEndQuote()) >= 0) {
+ return field;
+ }
+
+ if (isReservedWord(field) && quoteReservedWords()) {
+ return handleCase(getStartQuote() + field + getEndQuote());
+ } else {
+ if (databaseInterface.isQuoteAllFields()
+ || hasSpacesInField(field) || hasSpecialCharInField(field) || hasDotInField(field)) {
+ return getStartQuote() + field + getEndQuote();
+ } else {
+ return field;
+ }
+ }
+ }
+
+ private String handleCase(String field) {
+ if (preserveReservedCase()) {
+ return field;
+ } else {
+ if (databaseInterface.isDefaultingToUppercase()) {
+ return field.toUpperCase();
+ } else {
+ return field.toLowerCase();
+ }
+ }
+ }
+
+ /**
+ * Determines whether or not this field is in need of quoting: getFeatureSummary() {
+ List list = new ArrayList();
+ RowMetaAndData r = null;
+ final String par = "Parameter";
+ final String val = "Value";
+
+ ValueMetaInterface testValue = new ValueMetaString("FIELD");
+ testValue.setLength(30);
+
+ if (databaseInterface != null) {
+ // Type of database
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Database type");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getPluginId());
+ list.add(r);
+ // Type of access
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Access type");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getAccessTypeDesc());
+ list.add(r);
+ // Name of database
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Database name");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getDatabaseName());
+ list.add(r);
+ // server host name
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Server hostname");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getHostname());
+ list.add(r);
+ // Port number
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Service port");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getDatabasePortNumberString());
+ list.add(r);
+ // Username
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Username");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getUsername());
+ list.add(r);
+ // Informix server
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Informix server name");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getServername());
+ list.add(r);
+ // Other properties...
+ Enumeration keys = getAttributes().keys();
+ while (keys.hasMoreElements()) {
+ String key = (String) keys.nextElement();
+ String value = getAttributes().getProperty(key);
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Extra attribute [" + key + "]");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, value);
+ list.add(r);
+ }
+
+ // driver class
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Driver class");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getDriverClass());
+ list.add(r);
+ // URL
+ String pwd = getPassword();
+ setPassword("password"); // Don't give away the password in the URL!
+ String url = "";
+ try {
+ url = getURL();
+ } catch (Exception e) {
+ url = "";
+ } // SAP etc.
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "URL");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, url);
+ list.add(r);
+ setPassword(pwd);
+ // SQL: Next sequence value
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "SQL: next sequence value");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getSeqNextvalSQL("SEQUENCE"));
+ list.add(r);
+ // is set fetch size supported
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supported: set fetch size");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, isFetchSizeSupported() ? "Y" : "N");
+ list.add(r);
+ // needs place holder for auto increment
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "auto increment field needs placeholder");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, needsPlaceHolder() ? "Y" : "N");
+ list.add(r);
+ // Sum function
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "SUM aggregate function");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getFunctionSum());
+ list.add(r);
+ // Avg function
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "AVG aggregate function");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getFunctionAverage());
+ list.add(r);
+ // Minimum function
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "MIN aggregate function");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getFunctionMinimum());
+ list.add(r);
+ // Maximum function
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "MAX aggregate function");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getFunctionMaximum());
+ list.add(r);
+ // Count function
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "COUNT aggregate function");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getFunctionCount());
+ list.add(r);
+ // Schema-table combination
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Schema / Table combination");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getQuotedSchemaTableCombination("SCHEMA", "TABLE"));
+ list.add(r);
+ // Limit clause
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "LIMIT clause for 100 rows");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getLimitClause(100));
+ list.add(r);
+ // add column statement
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Add column statement");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getAddColumnStatement(
+ "TABLE", testValue, null, false, null, false));
+ list.add(r);
+ // drop column statement
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Drop column statement");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getDropColumnStatement(
+ "TABLE", testValue, null, false, null, false));
+ list.add(r);
+ // Modify column statement
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Modify column statement");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getModifyColumnStatement(
+ "TABLE", testValue, null, false, null, false));
+ list.add(r);
+
+ // List of reserved words
+ String reserved = "";
+ if (getReservedWords() != null) {
+ for (int i = 0; i < getReservedWords().length; i++) {
+ reserved += (i > 0 ? ", " : "") + getReservedWords()[i];
+ }
+ }
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "List of reserved words");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, reserved);
+ list.add(r);
+
+ // Quote reserved words?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Quote reserved words?");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, quoteReservedWords() ? "Y" : "N");
+ list.add(r);
+ // Start Quote
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "Start quote for reserved words");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getStartQuote());
+ list.add(r);
+ // End Quote
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "End quote for reserved words");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getEndQuote());
+ list.add(r);
+
+ // List of table types
+ String types = "";
+ String[] slist = getTableTypes();
+ if (slist != null) {
+ for (int i = 0; i < slist.length; i++) {
+ types += (i > 0 ? ", " : "") + slist[i];
+ }
+ }
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "List of JDBC table types");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, types);
+ list.add(r);
+
+ // List of view types
+ types = "";
+ slist = getViewTypes();
+ if (slist != null) {
+ for (int i = 0; i < slist.length; i++) {
+ types += (i > 0 ? ", " : "") + slist[i];
+ }
+ }
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "List of JDBC view types");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, types);
+ list.add(r);
+
+ // List of synonym types
+ types = "";
+ slist = getSynonymTypes();
+ if (slist != null) {
+ for (int i = 0; i < slist.length; i++) {
+ types += (i > 0 ? ", " : "") + slist[i];
+ }
+ }
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "List of JDBC synonym types");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, types);
+ list.add(r);
+
+ // Use schema-name to get list of tables?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "use schema name to get table list?");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, useSchemaNameForTableList() ? "Y" : "N");
+ list.add(r);
+ // supports view?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports views?");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsViews() ? "Y" : "N");
+ list.add(r);
+ // supports synonyms?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports synonyms?");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsSynonyms() ? "Y" : "N");
+ list.add(r);
+ // SQL: get list of procedures?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "SQL: list of procedures");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, getSQLListOfProcedures());
+ list.add(r);
+ // SQL: get truncate table statement?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "SQL: truncate table");
+ String truncateStatement = getTruncateTableStatement(null, "TABLE");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, truncateStatement != null
+ ? truncateStatement : "Not supported by this database type");
+ list.add(r);
+ // supports float rounding on update?
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports floating point rounding on update/insert");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsFloatRoundingOnUpdate() ? "Y" : "N");
+ list.add(r);
+ // supports time stamp to date conversion
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports timestamp-date conversion");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsTimeStampToDateConversion() ? "Y" : "N");
+ list.add(r);
+ // supports batch updates
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports batch updates");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsBatchUpdates() ? "Y" : "N");
+ list.add(r);
+ // supports boolean values
+ r = new RowMetaAndData();
+ r.addValue(par, ValueMetaInterface.TYPE_STRING, "supports boolean data type");
+ r.addValue(val, ValueMetaInterface.TYPE_STRING, supportsBooleanDataType() ? "Y" : "N");
+ list.add(r);
+ }
+
+ return list;
+ }
+
+ /**
+ * @return true if the database result sets support getTimeStamp() to retrieve date-time. (Date)
+ */
+ public boolean supportsTimeStampToDateConversion() {
+ return databaseInterface.supportsTimeStampToDateConversion();
+ }
+
+ /**
+ * @return true if the database JDBC driver supports batch updates For example Interbase doesn't support this!
+ */
+ public boolean supportsBatchUpdates() {
+ return databaseInterface.supportsBatchUpdates();
+ }
+
+ /**
+ * @return true if the database supports a boolean, bit, logical, ... datatype
+ */
+ public boolean supportsBooleanDataType() {
+ return databaseInterface.supportsBooleanDataType();
+ }
+
+ /**
+ * @param b Set to true if the database supports a boolean, bit, logical, ... datatype
+ */
+ public void setSupportsBooleanDataType(boolean b) {
+ databaseInterface.setSupportsBooleanDataType(b);
+ }
+
+ /**
+ * @return true if the database supports the Timestamp data type (nanosecond precision and all)
+ */
+ public boolean supportsTimestampDataType() {
+ return databaseInterface.supportsTimestampDataType();
+ }
+
+ /**
+ * @param b Set to true if the database supports the Timestamp data type (nanosecond precision and all)
+ */
+ public void setSupportsTimestampDataType(boolean b) {
+ databaseInterface.setSupportsTimestampDataType(b);
+ }
+
+ /**
+ * @return true if reserved words' case should be preserved
+ */
+ public boolean preserveReservedCase() {
+ return databaseInterface.preserveReservedCase();
+ }
+
+ /**
+ * @return true if reserved words' case should be preserved
+ */
+ public void setPreserveReservedCase(boolean b) {
+ databaseInterface.setPreserveReservedCase(b);
+ }
+
+ /**
+ * Changes the names of the fields to their quoted equivalent if this is needed
+ *
+ * @param fields The row of fields to change
+ */
+ public void quoteReservedWords(RowMetaInterface fields) {
+ for (int i = 0; i < fields.size(); i++) {
+ ValueMetaInterface v = fields.getValueMeta(i);
+ v.setName(quoteField(v.getName()));
+ }
+ }
+
+ /**
+ * @return a map of all the extra URL options you want to set.
+ */
+ public Map getExtraOptions() {
+ return databaseInterface.getExtraOptions();
+ }
+
+ /**
+ * @return true if the database supports connection options in the URL, false if they are put in a Properties object.
+ */
+ public boolean supportsOptionsInURL() {
+ return databaseInterface.supportsOptionsInURL();
+ }
+
+ /**
+ * @return extra help text on the supported options on the selected database platform.
+ */
+ public String getExtraOptionsHelpText() {
+ return databaseInterface.getExtraOptionsHelpText();
+ }
+
+ /**
+ * @return true if the database JDBC driver supports getBlob on the resultset. If not we must use getBytes() to get
+ * the data.
+ */
+ public boolean supportsGetBlob() {
+ return databaseInterface.supportsGetBlob();
+ }
+
+ /**
+ * @return The SQL to execute right after connecting
+ */
+ public String getConnectSQL() {
+ return databaseInterface.getConnectSQL();
+ }
+
+ /**
+ * @param sql The SQL to execute right after connecting
+ */
+ public void setConnectSQL(String sql) {
+ databaseInterface.setConnectSQL(sql);
+ }
+
+ /**
+ * @return true if the database supports setting the maximum number of return rows in a resultset.
+ */
+ public boolean supportsSetMaxRows() {
+ return databaseInterface.supportsSetMaxRows();
+ }
+
+ /**
+ * Verify the name of the database and if required, change it if it already exists in the list of databases.
+ *
+ * @param databases the databases to check against.
+ * @param oldname the old name of the database
+ * @return the new name of the database connection
+ */
+ public String verifyAndModifyDatabaseName(List databases, String oldname) {
+ String name = getName();
+ if (name.equalsIgnoreCase(oldname)) {
+ return name; // nothing to see here: move along!
+ }
+
+ int nr = 2;
+ while (DatabaseMeta.findDatabase(databases, getName()) != null) {
+ setName(name + " " + nr);
+ setDisplayName(name + " " + nr);
+ nr++;
+ }
+ return getName();
+ }
+
+ /**
+ * @return true if we want to use a database connection pool
+ */
+ public boolean isUsingConnectionPool() {
+ return databaseInterface.isUsingConnectionPool();
+ }
+
+ /**
+ * @param usePool true if we want to use a database connection pool
+ */
+ public void setUsingConnectionPool(boolean usePool) {
+ databaseInterface.setUsingConnectionPool(usePool);
+ }
+
+ /**
+ * @return the maximum pool size
+ */
+ public int getMaximumPoolSize() {
+ return databaseInterface.getMaximumPoolSize();
+ }
+
+ /**
+ * @param maximumPoolSize the maximum pool size
+ */
+ public void setMaximumPoolSize(int maximumPoolSize) {
+ databaseInterface.setMaximumPoolSize(maximumPoolSize);
+ }
+
+ /**
+ * @return the initial pool size
+ */
+ public int getInitialPoolSize() {
+ return databaseInterface.getInitialPoolSize();
+ }
+
+ /**
+ * @param initalPoolSize the initial pool size
+ */
+ public void setInitialPoolSize(int initalPoolSize) {
+ databaseInterface.setInitialPoolSize(initalPoolSize);
+ }
+
+ /**
+ * @return true if the connection contains partitioning information
+ */
+ public boolean isPartitioned() {
+ return databaseInterface.isPartitioned();
+ }
+
+ /**
+ * @param partitioned true if the connection is set to contain partitioning information
+ */
+ public void setPartitioned(boolean partitioned) {
+ databaseInterface.setPartitioned(partitioned);
+ }
+
+ /**
+ * @return the available partition/host/databases/port combinations in the cluster
+ */
+ public PartitionDatabaseMeta[] getPartitioningInformation() {
+ if (!isPartitioned()) {
+ return new PartitionDatabaseMeta[]{};
+ }
+ return databaseInterface.getPartitioningInformation();
+ }
+
+ /**
+ * @param partitionInfo the available partition/host/databases/port combinations in the cluster
+ */
+ public void setPartitioningInformation(PartitionDatabaseMeta[] partitionInfo) {
+ databaseInterface.setPartitioningInformation(partitionInfo);
+ }
+
+ /**
+ * Finds the partition metadata for the given partition iD
+ *
+ * @param partitionId The partition ID to look for
+ * @return the partition database metadata or null if nothing was found.
+ */
+ public PartitionDatabaseMeta getPartitionMeta(String partitionId) {
+ PartitionDatabaseMeta[] partitionInfo = getPartitioningInformation();
+ for (int i = 0; i < partitionInfo.length; i++) {
+ if (partitionInfo[i].getPartitionId().equals(partitionId)) {
+ return partitionInfo[i];
+ }
+ }
+ return null;
+ }
+
+ public Properties getConnectionPoolingProperties() {
+ return databaseInterface.getConnectionPoolingProperties();
+ }
+
+ public void setConnectionPoolingProperties(Properties properties) {
+ databaseInterface.setConnectionPoolingProperties(properties);
+ }
+
+ public String getSQLTableExists(String tablename) {
+ return databaseInterface.getSQLTableExists(tablename);
+ }
+
+ public String getSQLColumnExists(String columnname, String tablename) {
+ return databaseInterface.getSQLColumnExists(columnname, tablename);
+ }
+
+ public boolean needsToLockAllTables() {
+ return databaseInterface.needsToLockAllTables();
+ }
+
+ /**
+ * @return true if the database is streaming results (normally this is an option just for MySQL).
+ */
+ public boolean isStreamingResults() {
+ return databaseInterface.isStreamingResults();
+ }
+
+ /**
+ * @param useStreaming true if we want the database to stream results (normally this is an option just for MySQL).
+ */
+ public void setStreamingResults(boolean useStreaming) {
+ databaseInterface.setStreamingResults(useStreaming);
+ }
+
+ /**
+ * @return true if all fields should always be quoted in db
+ */
+ public boolean isQuoteAllFields() {
+ return databaseInterface.isQuoteAllFields();
+ }
+
+ /**
+ * @param quoteAllFields true if all fields in DB should be quoted.
+ */
+ public void setQuoteAllFields(boolean quoteAllFields) {
+ databaseInterface.setQuoteAllFields(quoteAllFields);
+ }
+
+ /**
+ * @return true if all identifiers should be forced to lower case
+ */
+ public boolean isForcingIdentifiersToLowerCase() {
+ return databaseInterface.isForcingIdentifiersToLowerCase();
+ }
+
+ /**
+ * @param forceLowerCase true if all identifiers should be forced to lower case
+ */
+ public void setForcingIdentifiersToLowerCase(boolean forceLowerCase) {
+ databaseInterface.setForcingIdentifiersToLowerCase(forceLowerCase);
+ }
+
+ /**
+ * @return true if all identifiers should be forced to upper case
+ */
+ public boolean isForcingIdentifiersToUpperCase() {
+ return databaseInterface.isForcingIdentifiersToUpperCase();
+ }
+
+ /**
+ * @param forceUpperCase true if all identifiers should be forced to upper case
+ */
+ public void setForcingIdentifiersToUpperCase(boolean forceUpperCase) {
+ databaseInterface.setForcingIdentifiersToUpperCase(forceUpperCase);
+ }
+
+ /**
+ * Find a database with a certain name in an arraylist of databases.
+ *
+ * @param databases The ArrayList of databases
+ * @param dbname The name of the database connection
+ * @return The database object if one was found, null otherwise.
+ */
+ public static final DatabaseMeta findDatabase(List databases, String dbname) {
+ if (databases == null) {
+ return null;
+ }
+
+ for (int i = 0; i < databases.size(); i++) {
+ DatabaseMeta ci = (DatabaseMeta) databases.get(i);
+ if (ci.getName().equalsIgnoreCase(dbname)) {
+ return ci;
+ }
+ }
+ return null;
+ }
+
+ public static int indexOfName(String[] databaseNames, String name) {
+ if (databaseNames == null || name == null) {
+ return -1;
+ }
+
+ for (int i = 0; i < databaseNames.length; i++) {
+ String databaseName = databaseNames[i];
+ if (name.equalsIgnoreCase(databaseName)) {
+ return i;
+ }
+ }
+
+ return -1;
+ }
+
+ /**
+ * Find a database with a certain ID in an arraylist of databases.
+ *
+ * @param databases The ArrayList of databases
+ * @param id The id of the database connection
+ * @return The database object if one was found, null otherwise.
+ */
+ public static final DatabaseMeta findDatabase(List databases, ObjectId id) {
+ if (databases == null) {
+ return null;
+ }
+
+ for (DatabaseMeta ci : databases) {
+ if (ci.getObjectId() != null && ci.getObjectId().equals(id)) {
+ return ci;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void copyVariablesFrom(VariableSpace space) {
+ variables.copyVariablesFrom(space);
+ }
+
+ @Override
+ public String environmentSubstitute(String aString) {
+ return variables.environmentSubstitute(aString);
+ }
+
+ @Override
+ public String[] environmentSubstitute(String[] aString) {
+ return variables.environmentSubstitute(aString);
+ }
+
+ @Override
+ public String fieldSubstitute(String aString, RowMetaInterface rowMeta, Object[] rowData) throws KettleValueException {
+ return variables.fieldSubstitute(aString, rowMeta, rowData);
+ }
+
+ @Override
+ public VariableSpace getParentVariableSpace() {
+ return variables.getParentVariableSpace();
+ }
+
+ @Override
+ public void setParentVariableSpace(VariableSpace parent) {
+ variables.setParentVariableSpace(parent);
+ }
+
+ @Override
+ public String getVariable(String variableName, String defaultValue) {
+ return variables.getVariable(variableName, defaultValue);
+ }
+
+ @Override
+ public String getVariable(String variableName) {
+ return variables.getVariable(variableName);
+ }
+
+ @Override
+ public boolean getBooleanValueOfVariable(String variableName, boolean defaultValue) {
+ if (!Const.isEmpty(variableName)) {
+ String value = environmentSubstitute(variableName);
+ if (!Const.isEmpty(value)) {
+ return ValueMetaBase.convertStringToBoolean(value);
+ }
+ }
+ return defaultValue;
+ }
+
+ @Override
+ public void initializeVariablesFrom(VariableSpace parent) {
+ variables.initializeVariablesFrom(parent);
+ }
+
+ @Override
+ public String[] listVariables() {
+ return variables.listVariables();
+ }
+
+ @Override
+ public void setVariable(String variableName, String variableValue) {
+ variables.setVariable(variableName, variableValue);
+ }
+
+ @Override
+ public void shareVariablesWith(VariableSpace space) {
+ variables = space;
+ }
+
+ @Override
+ public void injectVariables(Map prop) {
+ variables.injectVariables(prop);
+ }
+
+ /**
+ * @return the SQL Server instance
+ */
+ public String getSQLServerInstance() {
+ // This is also covered/persisted by JDBC option MS SQL Server / instancename /
+ // We want to return
+ // --> MSSQL.instancename
+ return getExtraOptions().get("MSSQL.instance");
+ }
+
+ /**
+ * @param instanceName the SQL Server instance
+ */
+ public void setSQLServerInstance(String instanceName) {
+ // This is also covered/persisted by JDBC option MS SQL Server / instancename /
+ // We want to return set
+ // --> MSSQL.instancename
+ addExtraOption("MSSQL", "instance", instanceName);
+ }
+
+ /**
+ * @return true if the Microsoft SQL server uses two decimals (..) to separate schema and table (default==false).
+ */
+ public boolean isUsingDoubleDecimalAsSchemaTableSeparator() {
+ return databaseInterface.isUsingDoubleDecimalAsSchemaTableSeparator();
+ }
+
+ /**
+ * @param useDoubleDecimalSeparator true if we want the database to stream results (normally this is an option just for MySQL).
+ */
+ public void setUsingDoubleDecimalAsSchemaTableSeparator(boolean useDoubleDecimalSeparator) {
+ databaseInterface.setUsingDoubleDecimalAsSchemaTableSeparator(useDoubleDecimalSeparator);
+ }
+
+ /**
+ * @return true if this database needs a transaction to perform a query (auto-commit turned off).
+ */
+ public boolean isRequiringTransactionsOnQueries() {
+ return databaseInterface.isRequiringTransactionsOnQueries();
+ }
+
+ public String testConnection() {
+
+ StringBuffer report = new StringBuffer();
+
+ // If the plug-in needs to provide connection information, we ask the DatabaseInterface...
+ //
+ try {
+ DatabaseFactoryInterface factory = getDatabaseFactory();
+ return factory.getConnectionTestReport(this);
+ } catch (ClassNotFoundException e) {
+ report
+ .append(BaseMessages.getString(PKG, "BaseDatabaseMeta.TestConnectionReportNotImplemented.Message"))
+ .append(Const.CR);
+ report.append(BaseMessages.getString(PKG, "DatabaseMeta.report.ConnectionError", getName())
+ + e.toString() + Const.CR);
+ report.append(Const.getStackTracker(e) + Const.CR);
+ } catch (Exception e) {
+ report.append(BaseMessages.getString(PKG, "DatabaseMeta.report.ConnectionError", getName())
+ + e.toString() + Const.CR);
+ report.append(Const.getStackTracker(e) + Const.CR);
+ }
+ return report.toString();
+ }
+
+ public DatabaseFactoryInterface getDatabaseFactory() throws Exception {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ PluginInterface plugin = registry.getPlugin(DatabasePluginType.class, databaseInterface.getPluginId());
+ if (plugin == null) {
+ throw new KettleDatabaseException("database type with plugin id ["
+ + databaseInterface.getPluginId() + "] couldn't be found!");
+ }
+
+ ClassLoader loader = registry.getClassLoader(plugin);
+
+ Class clazz = Class.forName(databaseInterface.getDatabaseFactoryName(), true, loader);
+ return (DatabaseFactoryInterface) clazz.newInstance();
+ }
+
+ public String getPreferredSchemaName() {
+ return databaseInterface.getPreferredSchemaName();
+ }
+
+ public void setPreferredSchemaName(String preferredSchemaName) {
+ databaseInterface.setPreferredSchemaName(preferredSchemaName);
+ }
+
+ /**
+ * Not used in this case, simply return root /
+ */
+ @Override
+ public RepositoryDirectoryInterface getRepositoryDirectory() {
+ return new RepositoryDirectory();
+ }
+
+ @Override
+ public void setRepositoryDirectory(RepositoryDirectoryInterface repositoryDirectory) {
+ throw new RuntimeException("Setting a directory on a database connection is not supported");
+ }
+
+ @Override
+ public RepositoryObjectType getRepositoryElementType() {
+ return REPOSITORY_ELEMENT_TYPE;
+ }
+
+ @Override
+ public ObjectRevision getObjectRevision() {
+ return objectRevision;
+ }
+
+ @Override
+ public void setObjectRevision(ObjectRevision objectRevision) {
+ this.objectRevision = objectRevision;
+ }
+
+ @Override
+ public String getDescription() {
+ // NOT USED
+ return null;
+ }
+
+ @Override
+ public void setDescription(String description) {
+ // NOT USED
+ }
+
+ public boolean supportsSequenceNoMaxValueOption() {
+ return databaseInterface.supportsSequenceNoMaxValueOption();
+ }
+
+ public boolean requiresCreateTablePrimaryKeyAppend() {
+ return databaseInterface.requiresCreateTablePrimaryKeyAppend();
+ }
+
+ public boolean requiresCastToVariousForIsNull() {
+ return databaseInterface.requiresCastToVariousForIsNull();
+ }
+
+ public boolean isDisplaySizeTwiceThePrecision() {
+ return databaseInterface.isDisplaySizeTwiceThePrecision();
+ }
+
+ public boolean supportsPreparedStatementMetadataRetrieval() {
+ return databaseInterface.supportsPreparedStatementMetadataRetrieval();
+ }
+
+ public boolean isSystemTable(String tableName) {
+ return databaseInterface.isSystemTable(tableName);
+ }
+
+ private boolean supportsNewLinesInSQL() {
+ return databaseInterface.supportsNewLinesInSQL();
+ }
+
+ public String getSQLListOfSchemas() {
+ return databaseInterface.getSQLListOfSchemas();
+ }
+
+ public int getMaxColumnsInIndex() {
+ return databaseInterface.getMaxColumnsInIndex();
+ }
+
+ public boolean supportsErrorHandlingOnBatchUpdates() {
+ return databaseInterface.supportsErrorHandlingOnBatchUpdates();
+ }
+
+ /**
+ * Get the SQL to insert a new empty unknown record in a dimension.
+ *
+ * @param schemaTable the schema-table name to insert into
+ * @param keyField The key field
+ * @param versionField the version field
+ * @return the SQL to insert the unknown record into the SCD.
+ */
+ public String getSQLInsertAutoIncUnknownDimensionRow(String schemaTable, String keyField, String versionField) {
+ return databaseInterface.getSQLInsertAutoIncUnknownDimensionRow(schemaTable, keyField, versionField);
+ }
+
+ /**
+ * @return true if this is a relational database you can explore. Return false for SAP, PALO, etc.
+ */
+ public boolean isExplorable() {
+ return databaseInterface.isExplorable();
+ }
+
+ /**
+ * @return The SQL on this database to get a list of sequences.
+ */
+ public String getSQLListOfSequences() {
+ return databaseInterface.getSQLListOfSequences();
+ }
+
+ public String quoteSQLString(String string) {
+ return databaseInterface.quoteSQLString(string);
+ }
+
+ /**
+ * @see DatabaseInterface#generateColumnAlias(int, String)
+ */
+ public String generateColumnAlias(int columnIndex, String suggestedName) {
+ return databaseInterface.generateColumnAlias(columnIndex, suggestedName);
+ }
+
+ public boolean isMySQLVariant() {
+ return databaseInterface.isMySQLVariant();
+ }
+
+ public Long getNextBatchId(Database ldb, String schemaName, String tableName, String fieldName) throws KettleDatabaseException {
+ return databaseInterface.getNextBatchId(this, ldb, schemaName, tableName, fieldName);
+ }
+
+ public Object getValueFromResultSet(ResultSet rs, ValueMetaInterface val, int i) throws KettleDatabaseException {
+ return databaseInterface.getValueFromResultSet(rs, val, i);
+ }
+
+ /**
+ * Marker used to determine if the DatabaseMeta should be allowed to be modified/saved. It does NOT prevent object
+ * modification.
+ *
+ * @return
+ */
+ public boolean isReadOnly() {
+ return readOnly;
+ }
+
+ /**
+ * Sets the marker used to determine if the DatabaseMeta should be allowed to be modified/saved. Setting to true does
+ * NOT prevent object modification.
+ *
+ * @return
+ */
+ public void setReadOnly(boolean readOnly) {
+ this.readOnly = readOnly;
+ }
+
+ public String getSequenceNoMaxValueOption() {
+ return databaseInterface.getSequenceNoMaxValueOption();
+ }
+
+ /**
+ * @return true if the database supports autoGeneratedKeys
+ */
+ public boolean supportsAutoGeneratedKeys() {
+ return databaseInterface.supportsAutoGeneratedKeys();
+ }
+
+
+ /**
+ * Customizes the ValueMetaInterface defined in the base
+ *
+ * @return String the create table statement
+ */
+ public String getCreateTableStatement() {
+ return databaseInterface.getCreateTableStatement();
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DataSourceLocator.java b/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DataSourceLocator.java
new file mode 100644
index 0000000..f881185
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DataSourceLocator.java
@@ -0,0 +1,142 @@
+/*! ******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+package org.pentaho.di.core.database.util;
+
+import org.osjava.sj.loader.SJDataSource;
+import org.pentaho.database.IDatabaseDialect;
+import org.pentaho.database.dialect.GenericDatabaseDialect;
+import org.pentaho.database.model.DatabaseAccessType;
+import org.pentaho.database.model.IDatabaseConnection;
+import org.pentaho.di.core.logging.LogChannelInterface;
+import org.pentaho.di.www.CarteSingleton;
+
+import javax.naming.Context;
+import javax.sql.DataSource;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Utility class for data source lookup.
+ *
+ * @author Zhichun Wu
+ */
+public final class DataSourceLocator {
+ private static boolean activated = false;
+
+ private static final AtomicBoolean changed = new AtomicBoolean(false);
+ private static final Map dataSources
+ = Collections.synchronizedMap(new HashMap());
+
+ private static final ServiceLoader dialectLoader = ServiceLoader.load(IDatabaseDialect.class);
+
+ private static DataSource buildDataSource(IDatabaseConnection conn) throws Exception {
+ String dbType = conn.getDatabaseType().getShortName();
+
+ SJDataSource ds = null;
+
+ for (IDatabaseDialect dialect : dialectLoader) {
+ // FIXME fallback to name/desc like Kettle did?
+ if (!dbType.equals(dialect.getDatabaseType().getShortName())) {
+ continue;
+ }
+
+ ds = new SJDataSource(dialect instanceof GenericDatabaseDialect
+ ? conn.getAttributes().get(GenericDatabaseDialect.ATTRIBUTE_CUSTOM_DRIVER_CLASS)
+ : dialect.getNativeDriver(),
+ dialect.supportsOptionsInURL() ? dialect.getURLWithExtraOptions(conn) : dialect.getURL(conn),
+ conn.getUsername(),
+ conn.getPassword(),
+ new Properties());
+ break;
+ }
+
+ return ds;
+ }
+
+ /**
+ * This method tries to import data sources defined in Kettle master time after time.
+ * It does nothing in non-cluster environment, but it addresses data source configuration issue in Kettle cluster:
+ * - zero data source configuration in Kettle slave servers - everything comes from master
+ * - cache data source configuration in memory for a while for better performance
+ * - update cached data source in case Kettle master changed configuration of a certain data source
+ *
+ * @param dsCache shared in-memory cache of data sources regardless where it comes from
+ * @param ctx naming context for binding / rebinding data source from master
+ * @param dsName name of the data source
+ */
+ static void importDataSourcesFromMaster(Map dsCache, Context ctx, String dsName) {
+ if (!activated) {
+ return;
+ }
+
+ // this is not truly thread-safe, as data sources might be updated in the same time
+ // however, it is worthy of doing this for better performance
+ if (changed.compareAndSet(true, false)) {
+ dsCache.putAll(dataSources);
+
+ LogChannelInterface log = CarteSingleton.getInstance().getLog();
+ log.logBasic("Data source cache refreshed successfully");
+ }
+ }
+
+ public static void activate() {
+ activated = true;
+ }
+
+ public static void deactivate() {
+ activated = false;
+ }
+
+
+ public static int updateDataSourceMappings(Map mapping) {
+ if (mapping == null || mapping.size() == 0) {
+ return 0;
+ }
+
+ LogChannelInterface log = CarteSingleton.getInstance().getLog();
+
+ int counter = 0;
+ for (Map.Entry entry : mapping.entrySet()) {
+ String dsName = entry.getKey();
+ IDatabaseConnection ds = entry.getValue();
+
+ if (ds.getAccessType() == DatabaseAccessType.NATIVE && ds.getDatabaseType() != null) {
+ try {
+ DataSource d = buildDataSource(ds);
+ if (d == null) {
+ log.logError("Bypass unsupported data source: " + dsName);
+ continue;
+ }
+
+ // it's better to use a separated map here but we should be just fine
+ dataSources.put(dsName, d);
+ counter++;
+ } catch (Exception e) {
+ log.logError("Failed to build data source: " + dsName, e);
+ }
+ }
+ }
+
+ if (counter > 0) {
+ changed.set(true);
+ }
+
+ return counter;
+ }
+
+ private DataSourceLocator() {
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DatabaseUtil.java b/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DatabaseUtil.java
new file mode 100644
index 0000000..cb74f84
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/core/database/util/DatabaseUtil.java
@@ -0,0 +1,199 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.core.database.util;
+
+import org.pentaho.di.core.Const;
+import org.pentaho.di.core.database.DataSourceNamingException;
+import org.pentaho.di.core.database.DataSourceProviderInterface;
+import org.pentaho.di.core.database.Database;
+import org.pentaho.di.i18n.BaseMessages;
+
+import javax.naming.Context;
+import javax.naming.InitialContext;
+import javax.naming.NamingException;
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Provides default implementation for looking data sources up in JNDI.
+ *
+ * @author mbatchel
+ */
+
+public class DatabaseUtil implements DataSourceProviderInterface {
+ private static Class PKG = Database.class; // for i18n purposes, needed by Translator2!!
+ private static Map FoundDS = Collections.synchronizedMap(new HashMap());
+
+ /**
+ * Clears cache of DataSources (For Unit test)
+ */
+ protected static void clearDSCache() {
+ FoundDS.clear();
+ }
+
+ /**
+ * Since JNDI is supported different ways in different app servers, it's nearly impossible to have a ubiquitous way to
+ * look up a datasource. This method is intended to hide all the lookups that may be required to find a jndi name.
+ *
+ * @param dsName The Datasource name
+ * @return DataSource if there is one bound in JNDI
+ * @throws NamingException
+ */
+ protected static DataSource getDataSourceFromJndi(String dsName, Context ctx) throws NamingException {
+ if (Const.isEmpty(dsName)) {
+ throw new NamingException(BaseMessages.getString(PKG, "DatabaseUtil.DSNotFound", String.valueOf(dsName)));
+ }
+
+ // Too bad there's no context for us to figure out which master initiated the data source lookup
+ // FIXME FoundDS is polluted here...
+ DataSourceLocator.importDataSourcesFromMaster(FoundDS, ctx, dsName);
+
+ Object foundDs = FoundDS.get(dsName);
+ if (foundDs != null) {
+ return (DataSource) foundDs;
+ }
+ Object lkup = null;
+ DataSource rtn = null;
+ NamingException firstNe = null;
+ // First, try what they ask for...
+ try {
+ lkup = ctx.lookup(dsName);
+ if (lkup instanceof DataSource) {
+ rtn = (DataSource) lkup;
+ FoundDS.put(dsName, rtn);
+ return rtn;
+ }
+ } catch (NamingException ignored) {
+ firstNe = ignored;
+ }
+ try {
+ // Needed this for Jboss
+ lkup = ctx.lookup("java:" + dsName);
+ if (lkup instanceof DataSource) {
+ rtn = (DataSource) lkup;
+ FoundDS.put(dsName, rtn);
+ return rtn;
+ }
+ } catch (NamingException ignored) {
+ // ignore
+ }
+ try {
+ // Tomcat
+ lkup = ctx.lookup("java:comp/env/jdbc/" + dsName);
+ if (lkup instanceof DataSource) {
+ rtn = (DataSource) lkup;
+ FoundDS.put(dsName, rtn);
+ return rtn;
+ }
+ } catch (NamingException ignored) {
+ // ignore
+ }
+ try {
+ // Others?
+ lkup = ctx.lookup("jdbc/" + dsName);
+ if (lkup instanceof DataSource) {
+ rtn = (DataSource) lkup;
+ FoundDS.put(dsName, rtn);
+ return rtn;
+ }
+ } catch (NamingException ignored) {
+ // ignore
+ }
+ if (firstNe != null) {
+ throw firstNe;
+ }
+ throw new NamingException(BaseMessages.getString(PKG, "DatabaseUtil.DSNotFound", dsName));
+ }
+
+ public static void closeSilently(Connection[] connections) {
+ if (connections == null || connections.length == 0) {
+ return;
+ }
+ for (Connection conn : connections) {
+ closeSilently(conn);
+ }
+ }
+
+ public static void closeSilently(Connection conn) {
+ if (conn == null) {
+ return;
+ }
+ try {
+ conn.close();
+ } catch (Throwable e) {
+ // omit
+ }
+ }
+
+ public static void closeSilently(Statement[] statements) {
+ if (statements == null || statements.length == 0) {
+ return;
+ }
+ for (Statement st : statements) {
+ closeSilently(st);
+ }
+ }
+
+ public static void closeSilently(Statement st) {
+ if (st == null) {
+ return;
+ }
+ try {
+ st.close();
+ } catch (Throwable e) {
+ // omit
+ }
+ }
+
+
+ /**
+ * Implementation of DatasourceProviderInterface.
+ */
+ @Override
+ public DataSource getNamedDataSource(String datasourceName) throws DataSourceNamingException {
+ try {
+ return DatabaseUtil.getDataSourceFromJndi(datasourceName, new InitialContext());
+ } catch (NamingException ex) {
+ throw new DataSourceNamingException(ex);
+ }
+ }
+
+ @Override
+ public DataSource getNamedDataSource(String datasourceName, DatasourceType type)
+ throws DataSourceNamingException {
+ if (type != null) {
+ switch (type) {
+ case JNDI:
+ return getNamedDataSource(datasourceName);
+ case POOLED:
+ throw new UnsupportedOperationException(
+ getClass().getName() + " does not support providing pooled data sources");
+ }
+ }
+ throw new IllegalArgumentException("Unsupported data source type: " + type);
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LogChannelFileWriter.java b/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LogChannelFileWriter.java
new file mode 100644
index 0000000..4779ca9
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LogChannelFileWriter.java
@@ -0,0 +1,210 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.core.logging;
+
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.vfs.KettleVFS;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class takes care of polling the central log store for new log messages belonging to a certain log channel ID
+ * (and children). The log lines will be written to a logging file.
+ *
+ * @author matt
+ */
+public class LogChannelFileWriter {
+ private String logChannelId;
+ private FileObject logFile;
+ private boolean appending;
+ private int pollingInterval;
+
+ private AtomicBoolean active;
+ private KettleException exception;
+ private int lastBufferLineNr;
+ protected OutputStream logFileOutputStream;
+
+ /**
+ * Create a new log channel file writer
+ *
+ * @param logChannelId The log channel (+children) to write to the log file
+ * @param logFile The logging file to write to
+ * @param appending set to true if you want to append to an existing file
+ * @param pollingInterval The polling interval in milliseconds.
+ * @throws KettleException in case the specified log file can't be created.
+ */
+ public LogChannelFileWriter(String logChannelId, FileObject logFile, boolean appending, int pollingInterval) throws KettleException {
+ this.logChannelId = logChannelId;
+ this.logFile = logFile;
+ this.appending = appending;
+ this.pollingInterval = pollingInterval;
+
+ active = new AtomicBoolean(false);
+ lastBufferLineNr = KettleLogStore.getLastBufferLineNr();
+
+ // it's basic move to create the directory *before* creating log file
+ try {
+ FileObject parent = logFile == null ? null : logFile.getParent();
+ if (parent != null && !parent.exists()) {
+ parent.createFolder();
+ }
+ } catch (Exception e) {
+ // ignore this type of exception as eventually KettleVFS will handle this
+ }
+
+ try {
+ logFileOutputStream = KettleVFS.getOutputStream(logFile, appending);
+ } catch (IOException e) {
+ throw new KettleException("There was an error while trying to open file '" + logFile + "' for writing", e);
+ }
+ }
+
+ /**
+ * Create a new log channel file writer
+ *
+ * @param logChannelId The log channel (+children) to write to the log file
+ * @param logFile The logging file to write to
+ * @param appending set to true if you want to append to an existing file
+ * @throws KettleException in case the specified log file can't be created.
+ */
+ public LogChannelFileWriter(String logChannelId, FileObject logFile, boolean appending) throws KettleException {
+ this(logChannelId, logFile, appending, 1000);
+ }
+
+ /**
+ * Start the logging thread which will write log data from the specified log channel to the log file. In case of an
+ * error, the exception will be available with method getException().
+ */
+ public void startLogging() {
+
+ exception = null;
+ active.set(true);
+
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ try {
+
+ while (active.get() && exception == null) {
+ flush();
+ Thread.sleep(pollingInterval);
+ }
+ // When done, save the last bit as well...
+ flush();
+
+ } catch (Exception e) {
+ exception = new KettleException("There was an error logging to file '" + logFile + "'", e);
+ } finally {
+ try {
+ if (logFileOutputStream != null) {
+ logFileOutputStream.close();
+ logFileOutputStream = null;
+ }
+ } catch (Exception e) {
+ exception = new KettleException("There was an error closing log file file '" + logFile + "'", e);
+ }
+ }
+ }
+ });
+ thread.start();
+ }
+
+ public synchronized void flush() {
+ try {
+ int last = KettleLogStore.getLastBufferLineNr();
+ StringBuffer buffer = KettleLogStore.getAppender().getBuffer(logChannelId, false, lastBufferLineNr, last);
+ logFileOutputStream.write(buffer.toString().getBytes());
+ lastBufferLineNr = last;
+ logFileOutputStream.flush();
+ } catch (Exception e) {
+ exception = new KettleException("There was an error logging to file '" + logFile + "'", e);
+ }
+ }
+
+ public void stopLogging() {
+ flush();
+ active.set(false);
+ }
+
+ public KettleException getException() {
+ return exception;
+ }
+
+ /**
+ * @return the logChannelId
+ */
+ public String getLogChannelId() {
+ return logChannelId;
+ }
+
+ /**
+ * @param logChannelId the logChannelId to set
+ */
+ public void setLogChannelId(String logChannelId) {
+ this.logChannelId = logChannelId;
+ }
+
+ /**
+ * @return the logFile
+ */
+ public FileObject getLogFile() {
+ return logFile;
+ }
+
+ /**
+ * @param logFile the logFile to set
+ */
+ public void setLogFile(FileObject logFile) {
+ this.logFile = logFile;
+ }
+
+ /**
+ * @return the appending
+ */
+ public boolean isAppending() {
+ return appending;
+ }
+
+ /**
+ * @param appending the appending to set
+ */
+ public void setAppending(boolean appending) {
+ this.appending = appending;
+ }
+
+ /**
+ * @return the pollingInterval
+ */
+ public int getPollingInterval() {
+ return pollingInterval;
+ }
+
+ /**
+ * @param pollingInterval the pollingInterval to set
+ */
+ public void setPollingInterval(int pollingInterval) {
+ this.pollingInterval = pollingInterval;
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LoggingRegistry.java b/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LoggingRegistry.java
new file mode 100644
index 0000000..891c7bf
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/core/logging/LoggingRegistry.java
@@ -0,0 +1,255 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.core.logging;
+
+import org.pentaho.di.core.Const;
+import org.pentaho.di.core.util.EnvUtil;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class LoggingRegistry {
+ // OK, why not declare the followings as final?
+ private static final LoggingRegistry registry = new LoggingRegistry();
+ private static final int DEFAULT_MAX_SIZE = 10000;
+
+ private final Map map;
+ private final Map> childrenMap;
+ private final int maxSize;
+ private final Object syncObject = new Object();
+
+ private Date lastModificationTime;
+
+ private LoggingRegistry() {
+ this.map = new ConcurrentHashMap();
+ this.childrenMap = new ConcurrentHashMap>();
+
+ this.lastModificationTime = new Date();
+ this.maxSize = Const.toInt(EnvUtil.getSystemProperty("KETTLE_MAX_LOGGING_REGISTRY_SIZE"), DEFAULT_MAX_SIZE);
+ }
+
+ public static LoggingRegistry getInstance() {
+ return registry;
+ }
+
+ public String registerLoggingSource(Object object) {
+ synchronized (this.syncObject) {
+ // it does not make sense to me to check the setting every time registering a logging service,
+ // not to mention I once had the weird thread dump below(all threads were locked up):
+ // Thread 28307: (state = BLOCKED)
+ // - java.lang.Integer.parseInt(java.lang.String, int) @bci=4, line=542 (Compiled frame)
+ // - java.lang.Integer.parseInt(java.lang.String) @bci=3, line=615 (Compiled frame)
+ // - org.pentaho.di.core.Const.toInt(java.lang.String, int) @bci=1, line=1173 (Compiled frame)
+ // - org.pentaho.di.core.logging.LoggingRegistry.registerLoggingSource(java.lang.Object) @bci=16, line=63 (Compiled frame)
+ // ...
+
+ // this.maxSize = Const.toInt( EnvUtil.getSystemProperty( "KETTLE_MAX_LOGGING_REGISTRY_SIZE" ), 10000 );
+
+ LoggingObject loggingSource = new LoggingObject(object);
+
+ LoggingObjectInterface found = findExistingLoggingSource(loggingSource);
+ if (found != null) {
+ LoggingObjectInterface foundParent = found.getParent();
+ LoggingObjectInterface loggingSourceParent = loggingSource.getParent();
+ if (foundParent != null && loggingSourceParent != null) {
+ String foundParentLogChannelId = foundParent.getLogChannelId();
+ String sourceParentLogChannelId = loggingSourceParent.getLogChannelId();
+ if (foundParentLogChannelId != null && sourceParentLogChannelId != null
+ && foundParentLogChannelId.equals(sourceParentLogChannelId)) {
+ String foundLogChannelId = found.getLogChannelId();
+ if (foundLogChannelId != null) {
+ return foundLogChannelId;
+ }
+ }
+ }
+ }
+
+ String logChannelId = UUID.randomUUID().toString();
+ loggingSource.setLogChannelId(logChannelId);
+
+ this.map.put(logChannelId, loggingSource);
+
+ if (loggingSource.getParent() != null) {
+ String parentLogChannelId = loggingSource.getParent().getLogChannelId();
+ if (parentLogChannelId != null) {
+ List parentChildren = this.childrenMap.get(parentLogChannelId);
+ if (parentChildren == null) {
+ parentChildren = new ArrayList();
+ this.childrenMap.put(parentLogChannelId, parentChildren);
+ }
+ parentChildren.add(logChannelId);
+ }
+ }
+
+ this.lastModificationTime = new Date();
+ loggingSource.setRegistrationDate(this.lastModificationTime);
+
+ if ((this.maxSize > 0) && (this.map.size() > this.maxSize)) {
+ List all = new ArrayList(this.map.values());
+ Collections.sort(all, new Comparator() {
+ @Override
+ public int compare(LoggingObjectInterface o1, LoggingObjectInterface o2) {
+ if ((o1 == null) && (o2 != null)) {
+ return -1;
+ }
+ if ((o1 != null) && (o2 == null)) {
+ return 1;
+ }
+ if ((o1 == null) && (o2 == null)) {
+ return 0;
+ }
+ if (o1.getRegistrationDate() == null && o2.getRegistrationDate() != null) {
+ return -1;
+ }
+ if (o1.getRegistrationDate() != null && o2.getRegistrationDate() == null) {
+ return 1;
+ }
+ if (o1.getRegistrationDate() == null && o2.getRegistrationDate() == null) {
+ return 0;
+ }
+ return (o1.getRegistrationDate().compareTo(o2.getRegistrationDate()));
+ }
+ });
+ int cutCount = this.maxSize < 1000 ? this.maxSize : 1000;
+ for (int i = 0; i < cutCount; i++) {
+ LoggingObjectInterface toRemove = all.get(i);
+ this.map.remove(toRemove.getLogChannelId());
+ }
+ removeOrphans();
+ }
+ return logChannelId;
+ }
+ }
+
+ public LoggingObjectInterface findExistingLoggingSource(LoggingObjectInterface loggingObject) {
+ LoggingObjectInterface found = null;
+ for (LoggingObjectInterface verify : this.map.values()) {
+ if (loggingObject.equals(verify)) {
+ found = verify;
+ break;
+ }
+ }
+ return found;
+ }
+
+ public LoggingObjectInterface getLoggingObject(String logChannelId) {
+ return this.map.get(logChannelId);
+ }
+
+ public Map getMap() {
+ return this.map;
+ }
+
+ public List getLogChannelChildren(String parentLogChannelId) {
+ if (parentLogChannelId == null) {
+ return null;
+ }
+ List list = getLogChannelChildren(new ArrayList(), parentLogChannelId);
+ list.add(parentLogChannelId);
+ return list;
+ }
+
+ private List getLogChannelChildren(List children, String parentLogChannelId) {
+ synchronized (this.syncObject) {
+ List list = this.childrenMap.get(parentLogChannelId);
+ if (list == null) {
+ // Don't do anything, just return the input.
+ return children;
+ }
+
+ Iterator kids = list.iterator();
+ while (kids.hasNext()) {
+ String logChannelId = kids.next();
+
+ // Add the children recursively
+ getLogChannelChildren(children, logChannelId);
+
+ // Also add the current parent
+ children.add(logChannelId);
+ }
+ }
+
+ return children;
+ }
+
+ public Date getLastModificationTime() {
+ return this.lastModificationTime;
+ }
+
+ public String dump(boolean includeGeneral) {
+ StringBuffer out = new StringBuffer(50000);
+ for (LoggingObjectInterface o : this.map.values()) {
+ if ((includeGeneral) || (!o.getObjectType().equals(LoggingObjectType.GENERAL))) {
+ out.append(o.getContainerObjectId());
+ out.append("\t");
+ out.append(o.getLogChannelId());
+ out.append("\t");
+ out.append(o.getObjectType().name());
+ out.append("\t");
+ out.append(o.getObjectName());
+ out.append("\t");
+ out.append(o.getParent() != null ? o.getParent().getLogChannelId() : "-");
+ out.append("\t");
+ out.append(o.getParent() != null ? o.getParent().getObjectType().name() : "-");
+ out.append("\t");
+ out.append(o.getParent() != null ? o.getParent().getObjectName() : "-");
+ out.append("\n");
+ }
+ }
+ return out.toString();
+ }
+
+ /**
+ * For junit testing purposes
+ *
+ * @return ro items map
+ */
+ Map dumpItems() {
+ return Collections.unmodifiableMap(this.map);
+ }
+
+ /**
+ * For junit testing purposes
+ *
+ * @return ro parent-child relations map
+ */
+ Map> dumpChildren() {
+ return Collections.unmodifiableMap(this.childrenMap);
+ }
+
+ public void removeIncludingChildren(String logChannelId) {
+ synchronized (this.map) {
+ List children = getLogChannelChildren(logChannelId);
+ for (String child : children) {
+ this.map.remove(child);
+ }
+ this.map.remove(logChannelId);
+ removeOrphans();
+ }
+ }
+
+ public void removeOrphans() {
+ // Remove all orphaned children
+ this.childrenMap.keySet().retainAll(this.map.keySet());
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/job/Job.java b/pentaho-kettle/src/main/java/org/pentaho/di/job/Job.java
new file mode 100644
index 0000000..45af58c
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/job/Job.java
@@ -0,0 +1,2428 @@
+//CHECKSTYLE:FileLength:OFF
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.job;
+
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.*;
+import org.pentaho.di.core.database.Database;
+import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.exception.KettleDatabaseException;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.exception.KettleJobException;
+import org.pentaho.di.core.exception.KettleValueException;
+import org.pentaho.di.core.extension.ExtensionPointHandler;
+import org.pentaho.di.core.extension.KettleExtensionPoint;
+import org.pentaho.di.core.gui.JobTracker;
+import org.pentaho.di.core.logging.*;
+import org.pentaho.di.core.parameters.DuplicateParamException;
+import org.pentaho.di.core.parameters.NamedParams;
+import org.pentaho.di.core.parameters.NamedParamsDefault;
+import org.pentaho.di.core.parameters.UnknownParamException;
+import org.pentaho.di.core.row.RowMetaInterface;
+import org.pentaho.di.core.row.ValueMeta;
+import org.pentaho.di.core.util.EnvUtil;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.variables.Variables;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.entries.job.JobEntryJob;
+import org.pentaho.di.job.entries.special.JobEntrySpecial;
+import org.pentaho.di.job.entries.trans.JobEntryTrans;
+import org.pentaho.di.job.entry.JobEntryCopy;
+import org.pentaho.di.job.entry.JobEntryInterface;
+import org.pentaho.di.repository.ObjectId;
+import org.pentaho.di.repository.ObjectRevision;
+import org.pentaho.di.repository.Repository;
+import org.pentaho.di.repository.RepositoryDirectoryInterface;
+import org.pentaho.di.resource.ResourceUtil;
+import org.pentaho.di.resource.TopLevelResource;
+import org.pentaho.di.trans.Trans;
+import org.pentaho.di.www.*;
+import org.pentaho.metastore.api.IMetaStore;
+
+import java.net.URLEncoder;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class executes a job as defined by a JobMeta object.
+ *
+ * The definition of a PDI job is represented by a JobMeta object. It is typically loaded from a .kjb file, a PDI
+ * repository, or it is generated dynamically. The declared parameters of the job definition are then queried using
+ * listParameters() and assigned values using calls to setParameterValue(..).
+ *
+ * @author Matt Casters
+ * @since 07-apr-2003
+ */
+public class Job extends Thread implements VariableSpace, NamedParams, HasLogChannelInterface, LoggingObjectInterface,
+ ExecutorInterface, ExtensionDataInterface {
+ private static Class PKG = Job.class; // for i18n purposes, needed by Translator2!!
+
+ public static final String CONFIGURATION_IN_EXPORT_FILENAME = "__job_execution_configuration__.xml";
+
+ private LogChannelInterface log;
+
+ private LogLevel logLevel = DefaultLogLevel.getLogLevel();
+
+ private String containerObjectId;
+
+ private JobMeta jobMeta;
+
+ private int logCommitSize = 10;
+
+ private Repository rep;
+
+ private AtomicInteger errors;
+
+ private VariableSpace variables = new Variables();
+
+ /**
+ * The job that's launching this (sub-) job. This gives us access to the whole chain, including the parent variables,
+ * etc.
+ */
+ protected Job parentJob;
+
+ /**
+ * The parent transformation
+ */
+ protected Trans parentTrans;
+
+ /**
+ * The parent logging interface to reference
+ */
+ private LoggingObjectInterface parentLoggingObject;
+
+ /**
+ * Keep a list of the job entries that were executed. org.pentaho.di.core.logging.CentralLogStore.getInstance()
+ */
+ private JobTracker jobTracker;
+
+ /**
+ * A flat list of results in THIS job, in the order of execution of job entries
+ */
+ private final LinkedList jobEntryResults = new LinkedList();
+
+ private Date startDate, endDate, currentDate, logDate, depDate;
+
+ private AtomicBoolean active;
+
+ private AtomicBoolean stopped;
+
+ private long batchId;
+
+ /**
+ * This is the batch ID that is passed from job to job to transformation, if nothing is passed, it's the job's batch
+ * id
+ */
+ private long passedBatchId;
+
+ /**
+ * The rows that were passed onto this job by a previous transformation. These rows are passed onto the first job
+ * entry in this job (on the result object)
+ */
+ private List sourceRows;
+
+ /**
+ * The result of the job, after execution.
+ */
+ private Result result;
+
+ private AtomicBoolean initialized;
+
+ private boolean interactive;
+
+ private List jobListeners;
+
+ private List jobEntryListeners;
+
+ private List delegationListeners;
+
+ private Map activeJobEntryTransformations;
+
+ private Map activeJobEntryJobs;
+
+ /**
+ * Parameters of the job.
+ */
+ private NamedParams namedParams = new NamedParamsDefault();
+
+ private AtomicBoolean finished;
+
+ private SocketRepository socketRepository;
+
+ private int maxJobEntriesLogged;
+
+ private JobEntryCopy startJobEntryCopy;
+ private Result startJobEntryResult;
+
+ private String executingServer;
+
+ private String executingUser;
+
+ private String transactionId;
+
+ private Map extensionDataMap;
+
+ /**
+ * The command line arguments for the job.
+ */
+ protected String[] arguments;
+
+ /**
+ * Instantiates a new job.
+ *
+ * @param name the name
+ * @param file the file
+ * @param args the args
+ */
+ public Job(String name, String file, String[] args) {
+ this();
+ jobMeta = new JobMeta();
+
+ if (name != null) {
+ setName(name + " (" + super.getName() + ")");
+ }
+ jobMeta.setName(name);
+ jobMeta.setFilename(file);
+ this.arguments = args;
+
+ init();
+ this.log = new LogChannel(this);
+ }
+
+ /**
+ * Initializes the Job.
+ */
+ public void init() {
+ jobListeners = new ArrayList();
+ jobEntryListeners = new ArrayList();
+ delegationListeners = new ArrayList();
+
+ activeJobEntryTransformations = new HashMap();
+ activeJobEntryJobs = new HashMap();
+
+ extensionDataMap = new HashMap();
+
+ active = new AtomicBoolean(false);
+ stopped = new AtomicBoolean(false);
+ jobTracker = new JobTracker(jobMeta);
+ synchronized (jobEntryResults) {
+ jobEntryResults.clear();
+ }
+ initialized = new AtomicBoolean(false);
+ finished = new AtomicBoolean(false);
+ errors = new AtomicInteger(0);
+ batchId = -1;
+ passedBatchId = -1;
+ maxJobEntriesLogged = Const.toInt(EnvUtil.getSystemProperty(Const.KETTLE_MAX_JOB_ENTRIES_LOGGED), 1000);
+
+ result = null;
+ startJobEntryCopy = null;
+ startJobEntryResult = null;
+
+ this.setDefaultLogCommitSize();
+ }
+
+ /**
+ * Sets the default log commit size.
+ */
+ private void setDefaultLogCommitSize() {
+ String propLogCommitSize = this.getVariable("pentaho.log.commit.size");
+ if (propLogCommitSize != null) {
+ // override the logCommit variable
+ try {
+ logCommitSize = Integer.parseInt(propLogCommitSize);
+ } catch (Exception ignored) {
+ logCommitSize = 10; // ignore parsing error and default to 10
+ }
+ }
+ }
+
+ /**
+ * Instantiates a new job.
+ *
+ * @param repository the repository
+ * @param jobMeta the job meta
+ */
+ public Job(Repository repository, JobMeta jobMeta) {
+ this(repository, jobMeta, null);
+ }
+
+ /**
+ * Instantiates a new job.
+ *
+ * @param repository the repository
+ * @param jobMeta the job meta
+ * @param parentLogging the parent logging
+ */
+ public Job(Repository repository, JobMeta jobMeta, LoggingObjectInterface parentLogging) {
+ this.rep = repository;
+ this.jobMeta = jobMeta;
+ this.parentLoggingObject = parentLogging;
+
+ init();
+
+ jobTracker = new JobTracker(jobMeta);
+
+ this.log = new LogChannel(this, parentLogging);
+ this.logLevel = log.getLogLevel();
+ this.containerObjectId = log.getContainerObjectId();
+ }
+
+ /**
+ * Empty constructor, for Class.newInstance()
+ */
+ public Job() {
+ init();
+ this.log = new LogChannel(this);
+ this.logLevel = log.getLogLevel();
+ }
+
+ /**
+ * Gets the name property of the JobMeta property.
+ *
+ * @return String name for the JobMeta
+ */
+ @Override
+ public String toString() {
+ if (jobMeta == null || Const.isEmpty(jobMeta.getName())) {
+ return getName();
+ } else {
+ return jobMeta.getName();
+ }
+ }
+
+ /**
+ * Creates the job with new class loader.
+ *
+ * @return the job
+ * @throws KettleException the kettle exception
+ */
+ public static final Job createJobWithNewClassLoader() throws KettleException {
+ try {
+ // Load the class.
+ Class jobClass = Const.createNewClassLoader().loadClass(Job.class.getName());
+
+ // create the class
+ // Try to instantiate this one...
+ Job job = (Job) jobClass.newInstance();
+
+ // Done!
+ return job;
+ } catch (Exception e) {
+ String message = BaseMessages.getString(PKG, "Job.Log.ErrorAllocatingNewJob", e.toString());
+ throw new KettleException(message, e);
+ }
+ }
+
+ /**
+ * Gets the jobname.
+ *
+ * @return the jobname
+ */
+ public String getJobname() {
+ if (jobMeta == null) {
+ return null;
+ }
+
+ return jobMeta.getName();
+ }
+
+ /**
+ * Sets the repository.
+ *
+ * @param rep the new repository
+ */
+ public void setRepository(Repository rep) {
+ this.rep = rep;
+ }
+
+ /**
+ * Threads main loop: called by Thread.start();
+ */
+ public void run() {
+
+ ExecutorService heartbeat = null; // this job's heartbeat scheduled executor
+
+ try {
+ stopped = new AtomicBoolean(false);
+ finished = new AtomicBoolean(false);
+ initialized = new AtomicBoolean(true);
+
+ // Create a new variable name space as we want jobs to have their own set of variables.
+ // initialize from parentJob or null
+ //
+ variables.initializeVariablesFrom(parentJob);
+ setInternalKettleVariables(variables);
+ copyParametersFrom(jobMeta);
+ activateParameters();
+
+ // Run the job
+ //
+ fireJobStartListeners();
+
+ heartbeat = startHeartbeat(getHeartbeatIntervalInSeconds());
+
+ result = execute();
+ } catch (Throwable je) {
+ log.logError(BaseMessages.getString(PKG, "Job.Log.ErrorExecJob", je.getMessage()), je);
+ // log.logError(Const.getStackTracker(je));
+ //
+ // we don't have result object because execute() threw a curve-ball.
+ // So we create a new error object.
+ //
+ result = new Result();
+ result.setNrErrors(1L);
+ result.setResult(false);
+ addErrors(1); // This can be before actual execution
+
+ emergencyWriteJobTracker(result);
+
+ active.set(false);
+ finished.set(true);
+ stopped.set(false);
+ } finally {
+ try {
+
+ shutdownHeartbeat(heartbeat);
+
+ ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.JobFinish.id, this);
+
+ fireJobFinishListeners();
+ } catch (KettleException e) {
+ result.setNrErrors(1);
+ result.setResult(false);
+ log.logError(BaseMessages.getString(PKG, "Job.Log.ErrorExecJob", e.getMessage()), e);
+
+ emergencyWriteJobTracker(result);
+ }
+ }
+ }
+
+ private void emergencyWriteJobTracker(Result res) {
+ JobEntryResult jerFinalResult =
+ new JobEntryResult(res, this.getLogChannelId(), BaseMessages.getString(PKG, "Job.Comment.JobFinished"), null,
+ null, 0, null);
+ JobTracker finalTrack = new JobTracker(this.getJobMeta(), jerFinalResult);
+ // jobTracker is up to date too.
+ this.jobTracker.addJobTracker(finalTrack);
+ }
+
+ /**
+ * Execute a job without previous results. This is a job entry point (not recursive) Then launch all the job listeners and call the jobFinished method for each. threads = new ArrayList();
+ // next 2 lists is being modified concurrently so must be synchronized for this case.
+ final Queue threadResults = new ConcurrentLinkedQueue();
+ final Queue threadExceptions = new ConcurrentLinkedQueue();
+ final List threadEntries = new ArrayList();
+
+ // Launch only those where the hop indicates true or false
+ //
+ int nrNext = jobMeta.findNrNextJobEntries(jobEntryCopy);
+ for (int i = 0; i < nrNext && !isStopped(); i++) {
+ // The next entry is...
+ final JobEntryCopy nextEntry = jobMeta.findNextJobEntry(jobEntryCopy, i);
+
+ // See if we need to execute this...
+ final JobHopMeta hi = jobMeta.findJobHop(jobEntryCopy, nextEntry);
+
+ // The next comment...
+ final String nextComment;
+ if (hi.isUnconditional()) {
+ nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedUnconditional");
+ } else {
+ if (newResult.getResult()) {
+ nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedSuccess");
+ } else {
+ nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedFailure");
+ }
+ }
+
+ //
+ // If the link is unconditional, execute the next job entry (entries).
+ // If the start point was an evaluation and the link color is correct:
+ // green or red, execute the next job entry...
+ //
+ if (hi.isUnconditional() || (jobEntryCopy.evaluates() && (!(hi.getEvaluation() ^ newResult
+ .getResult())))) {
+ // Start this next step!
+ if (log.isBasic()) {
+ log.logBasic(BaseMessages.getString(PKG, "Job.Log.StartingEntry", nextEntry.getName()));
+ }
+
+ // Pass along the previous result, perhaps the next job can use it...
+ // However, set the number of errors back to 0 (if it should be reset)
+ // When an evaluation is executed the errors e.g. should not be reset.
+ if (nextEntry.resetErrorsBeforeExecution()) {
+ newResult.setNrErrors(0);
+ }
+
+ // Now execute!
+ //
+ // if (we launch in parallel, fire the execution off in a new thread...
+ //
+ if (jobEntryCopy.isLaunchingInParallel()) {
+ threadEntries.add(nextEntry);
+
+ Runnable runnable = new Runnable() {
+ public void run() {
+ try {
+ Result threadResult = execute(nr + 1, newResult, nextEntry, jobEntryCopy, nextComment);
+ threadResults.add(threadResult);
+ } catch (Throwable e) {
+ log.logError(Const.getStackTracker(e));
+ threadExceptions.add(new KettleException(BaseMessages.getString(PKG, "Job.Log.UnexpectedError",
+ nextEntry.toString()), e));
+ Result threadResult = new Result();
+ threadResult.setResult(false);
+ threadResult.setNrErrors(1L);
+ threadResults.add(threadResult);
+ }
+ }
+ };
+ Thread thread = new Thread(runnable);
+ threads.add(thread);
+ thread.start();
+ if (log.isBasic()) {
+ log.logBasic(BaseMessages.getString(PKG, "Job.Log.LaunchedJobEntryInParallel", nextEntry.getName()));
+ }
+ } else {
+ try {
+ // Same as before: blocks until it's done
+ //
+ res = execute(nr + 1, newResult, nextEntry, jobEntryCopy, nextComment);
+ } catch (Throwable e) {
+ log.logError(Const.getStackTracker(e));
+ throw new KettleException(BaseMessages.getString(PKG, "Job.Log.UnexpectedError", nextEntry.toString()),
+ e);
+ }
+ if (log.isBasic()) {
+ log.logBasic(BaseMessages.getString(PKG, "Job.Log.FinishedJobEntry", nextEntry.getName(), res.getResult()
+ + ""));
+ }
+ }
+ }
+ }
+
+ // OK, if we run in parallel, we need to wait for all the job entries to
+ // finish...
+ //
+ if (jobEntryCopy.isLaunchingInParallel()) {
+ for (int i = 0; i < threads.size(); i++) {
+ Thread thread = threads.get(i);
+ JobEntryCopy nextEntry = threadEntries.get(i);
+
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ log.logError(jobMeta.toString(), BaseMessages.getString(PKG,
+ "Job.Log.UnexpectedErrorWhileWaitingForJobEntry", nextEntry.getName()));
+ threadExceptions.add(new KettleException(BaseMessages.getString(PKG,
+ "Job.Log.UnexpectedErrorWhileWaitingForJobEntry", nextEntry.getName()), e));
+ }
+ }
+ // if(log.isBasic()) log.logBasic(BaseMessages.getString(PKG,
+ // "Job.Log.FinishedJobEntry",startpoint.getName(),res.getResult()+""));
+ }
+
+ // Perhaps we don't have next steps??
+ // In this case, return the previous result.
+ if (res == null) {
+ res = prevResult;
+ }
+
+ // See if there where any errors in the parallel execution
+ //
+ if (threadExceptions.size() > 0) {
+ res.setResult(false);
+ res.setNrErrors(threadExceptions.size());
+
+ for (KettleException e : threadExceptions) {
+ log.logError(jobMeta.toString(), e.getMessage(), e);
+ }
+
+ // Now throw the first Exception for good measure...
+ //
+ throw threadExceptions.poll();
+ }
+
+ // In parallel execution, we aggregate all the results, simply add them to
+ // the previous result...
+ //
+ for (Result threadResult : threadResults) {
+ res.add(threadResult);
+ }
+
+ // If there have been errors, logically, we need to set the result to
+ // "false"...
+ //
+ if (res.getNrErrors() > 0) {
+ res.setResult(false);
+ }
+
+ return res;
+ }
+
+ /**
+ * Wait until this job has finished.
+ */
+ public void waitUntilFinished() {
+ waitUntilFinished(-1L);
+ }
+
+ /**
+ * Wait until this job has finished.
+ *
+ * @param maxMiliseconds the maximum number of ms to wait
+ */
+ public void waitUntilFinished(long maxMiliseconds) {
+ long time = 0L;
+ while (isAlive() && (time < maxMiliseconds || maxMiliseconds <= 0)) {
+ try {
+ Thread.sleep(1);
+ time += 1;
+ } catch (InterruptedException e) {
+ // Ignore sleep errors
+ }
+ }
+ }
+
+ /**
+ * Get the number of errors that happened in the job.
+ *
+ * @return nr of error that have occurred during execution. During execution of a job the number can change.
+ */
+ public int getErrors() {
+ return errors.get();
+ }
+
+ /**
+ * Set the number of occured errors to 0.
+ */
+ public void resetErrors() {
+ errors.set(0);
+ }
+
+ /**
+ * Add a number of errors to the total number of erros that occured during execution.
+ *
+ * @param nrToAdd nr of errors to add.
+ */
+ public void addErrors(int nrToAdd) {
+ if (nrToAdd > 0) {
+ errors.addAndGet(nrToAdd);
+ }
+ }
+
+ /**
+ * Handle logging at start
+ *
+ * @return true if it went OK.
+ * @throws KettleException
+ */
+ public boolean beginProcessing() throws KettleException {
+ currentDate = new Date();
+ logDate = new Date();
+ startDate = Const.MIN_DATE;
+ endDate = currentDate;
+
+ resetErrors();
+
+ final JobLogTable jobLogTable = jobMeta.getJobLogTable();
+ int intervalInSeconds = Const.toInt(environmentSubstitute(jobLogTable.getLogInterval()), -1);
+
+ if (jobLogTable.isDefined()) {
+
+ DatabaseMeta logcon = jobMeta.getJobLogTable().getDatabaseMeta();
+ String schemaName = environmentSubstitute(jobMeta.getJobLogTable().getActualSchemaName());
+ String tableName = environmentSubstitute(jobMeta.getJobLogTable().getActualTableName());
+ String schemaAndTable =
+ jobMeta.getJobLogTable().getDatabaseMeta().getQuotedSchemaTableCombination(schemaName, tableName);
+ Database ldb = new Database(this, logcon);
+ ldb.shareVariablesWith(this);
+ ldb.connect();
+ ldb.setCommit(logCommitSize);
+
+ try {
+ // See if we have to add a batch id...
+ Long id_batch = new Long(1);
+ if (jobMeta.getJobLogTable().isBatchIdUsed()) {
+ id_batch = logcon.getNextBatchId(ldb, schemaName, tableName, jobLogTable.getKeyField().getFieldName());
+ setBatchId(id_batch.longValue());
+ if (getPassedBatchId() <= 0) {
+ setPassedBatchId(id_batch.longValue());
+ }
+ }
+
+ Object[] lastr = ldb.getLastLogDate(schemaAndTable, jobMeta.getName(), true, LogStatus.END);
+ if (!Const.isEmpty(lastr)) {
+ Date last;
+ try {
+ last = ldb.getReturnRowMeta().getDate(lastr, 0);
+ } catch (KettleValueException e) {
+ throw new KettleJobException(BaseMessages.getString(PKG, "Job.Log.ConversionError", "" + tableName), e);
+ }
+ if (last != null) {
+ startDate = last;
+ }
+ }
+
+ depDate = currentDate;
+
+ ldb.writeLogRecord(jobMeta.getJobLogTable(), LogStatus.START, this, null);
+ if (!ldb.isAutoCommit()) {
+ ldb.commitLog(true, jobMeta.getJobLogTable());
+ }
+ ldb.disconnect();
+
+ // If we need to do periodic logging, make sure to install a timer for
+ // this...
+ //
+ if (intervalInSeconds > 0) {
+ final Timer timer = new Timer(getName() + " - interval logging timer");
+ TimerTask timerTask = new TimerTask() {
+ public void run() {
+ try {
+ endProcessing();
+ } catch (Exception e) {
+ log.logError(BaseMessages.getString(PKG, "Job.Exception.UnableToPerformIntervalLogging"), e);
+ // Also stop the show...
+ //
+
+ errors.incrementAndGet();
+ stopAll();
+ }
+ }
+ };
+ timer.schedule(timerTask, intervalInSeconds * 1000, intervalInSeconds * 1000);
+
+ addJobListener(new JobAdapter() {
+ public void jobFinished(Job job) {
+ timer.cancel();
+ }
+ });
+ }
+
+ // Add a listener at the end of the job to take of writing the final job
+ // log record...
+ //
+ addJobListener(new JobAdapter() {
+ public void jobFinished(Job job) throws KettleException {
+ try {
+ endProcessing();
+ } catch (KettleJobException e) {
+ log.logError(BaseMessages.getString(PKG, "Job.Exception.UnableToWriteToLoggingTable", jobLogTable
+ .toString()), e);
+ // do not skip exception here
+ // job is failed in case log database record is failed!
+ throw new KettleException(e);
+ }
+ }
+ });
+
+ } catch (KettleDatabaseException dbe) {
+ addErrors(1); // This is even before actual execution
+ throw new KettleJobException(BaseMessages.getString(PKG, "Job.Log.UnableToProcessLoggingStart", ""
+ + tableName), dbe);
+ } finally {
+ ldb.disconnect();
+ }
+ }
+
+ // If we need to write out the job entry logging information, do so at the end of the job:
+ //
+ JobEntryLogTable jobEntryLogTable = jobMeta.getJobEntryLogTable();
+ if (jobEntryLogTable.isDefined()) {
+ addJobListener(new JobAdapter() {
+ public void jobFinished(Job job) throws KettleException {
+ try {
+ writeJobEntryLogInformation();
+ } catch (KettleException e) {
+ throw new KettleException(BaseMessages.getString(PKG,
+ "Job.Exception.UnableToPerformJobEntryLoggingAtJobEnd"), e);
+ }
+ }
+ });
+ }
+
+ // If we need to write the log channel hierarchy and lineage information,
+ // add a listener for that too...
+ //
+ ChannelLogTable channelLogTable = jobMeta.getChannelLogTable();
+ if (channelLogTable.isDefined()) {
+ addJobListener(new JobAdapter() {
+
+ public void jobFinished(Job job) throws KettleException {
+ try {
+ writeLogChannelInformation();
+ } catch (KettleException e) {
+ throw new KettleException(BaseMessages.getString(PKG, "Job.Exception.UnableToPerformLoggingAtTransEnd"),
+ e);
+ }
+ }
+ });
+ }
+
+ JobExecutionExtension extension = new JobExecutionExtension(this, result, null, false);
+ ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.JobBeginProcessing.id, extension);
+
+ return true;
+ }
+
+ //
+ // Handle logging at end
+
+ /**
+ * End processing.
+ *
+ * @return true, if successful
+ * @throws KettleJobException the kettle job exception
+ */
+ private boolean endProcessing() throws KettleJobException {
+ LogStatus status;
+ if (!isActive()) {
+ if (isStopped()) {
+ status = LogStatus.STOP;
+ } else {
+ status = LogStatus.END;
+ }
+ } else {
+ status = LogStatus.RUNNING;
+ }
+ try {
+ if (errors.get() == 0 && result != null && !result.getResult()) {
+ errors.incrementAndGet();
+ }
+
+ logDate = new Date();
+
+ /*
+ * Sums errors, read, written, etc.
+ */
+
+ JobLogTable jobLogTable = jobMeta.getJobLogTable();
+ if (jobLogTable.isDefined()) {
+
+ writeLogTableInformation(jobLogTable, status);
+ }
+
+ return true;
+ } catch (Exception e) {
+ throw new KettleJobException(e); // In case something else goes wrong.
+ }
+ }
+
+ /**
+ * Writes information to Job Log table.
+ * Cleans old records, in case job is finished.
+ */
+ protected void writeLogTableInformation(JobLogTable jobLogTable, LogStatus status)
+ throws KettleJobException, KettleDatabaseException {
+ boolean cleanLogRecords = status.equals(LogStatus.END);
+ String tableName = jobLogTable.getActualTableName();
+ DatabaseMeta logcon = jobLogTable.getDatabaseMeta();
+
+ Database ldb = createDataBase(logcon);
+ ldb.shareVariablesWith(this);
+ try {
+ ldb.connect();
+ ldb.setCommit(logCommitSize);
+ ldb.writeLogRecord(jobLogTable, status, this, null);
+
+ if (cleanLogRecords) {
+ ldb.cleanupLogRecords(jobLogTable);
+ }
+
+ } catch (KettleDatabaseException dbe) {
+ addErrors(1);
+ throw new KettleJobException("Unable to end processing by writing log record to table " + tableName, dbe);
+ } finally {
+ if (!ldb.isAutoCommit()) {
+ ldb.commitLog(true, jobLogTable);
+ }
+ ldb.disconnect();
+ }
+ }
+
+ /**
+ * Write log channel information.
+ *
+ * @throws KettleException the kettle exception
+ */
+ protected void writeLogChannelInformation() throws KettleException {
+ Database db = null;
+ ChannelLogTable channelLogTable = jobMeta.getChannelLogTable();
+
+ // PDI-7070: If parent job has the same channel logging info, don't duplicate log entries
+ Job j = getParentJob();
+
+ if (j != null) {
+ if (channelLogTable.equals(j.getJobMeta().getChannelLogTable())) {
+ return;
+ }
+ }
+ // end PDI-7070
+
+ try {
+ db = new Database(this, channelLogTable.getDatabaseMeta());
+ db.shareVariablesWith(this);
+ db.connect();
+ db.setCommit(logCommitSize);
+
+ List loggingHierarchyList = getLoggingHierarchy();
+ for (LoggingHierarchy loggingHierarchy : loggingHierarchyList) {
+ db.writeLogRecord(channelLogTable, LogStatus.START, loggingHierarchy, null);
+ }
+
+ // Also time-out the log records in here...
+ //
+ db.cleanupLogRecords(channelLogTable);
+
+ } catch (Exception e) {
+ throw new KettleException(BaseMessages.getString(PKG,
+ "Trans.Exception.UnableToWriteLogChannelInformationToLogTable"), e);
+ } finally {
+ if (!db.isAutoCommit()) {
+ db.commit(true);
+ }
+ db.disconnect();
+ }
+ }
+
+ /**
+ * Write job entry log information.
+ *
+ * @throws KettleException the kettle exception
+ */
+ protected void writeJobEntryLogInformation() throws KettleException {
+ Database db = null;
+ JobEntryLogTable jobEntryLogTable = getJobMeta().getJobEntryLogTable();
+ try {
+ db = createDataBase(jobEntryLogTable.getDatabaseMeta());
+ db.shareVariablesWith(this);
+ db.connect();
+ db.setCommit(logCommitSize);
+
+ for (JobEntryCopy copy : getJobMeta().getJobCopies()) {
+ db.writeLogRecord(jobEntryLogTable, LogStatus.START, copy, this);
+ }
+
+ db.cleanupLogRecords(jobEntryLogTable);
+ } catch (Exception e) {
+ throw new KettleException(BaseMessages.getString(PKG, "Job.Exception.UnableToJobEntryInformationToLogTable"),
+ e);
+ } finally {
+ if (!db.isAutoCommit()) {
+ db.commitLog(true, jobEntryLogTable);
+ }
+ db.disconnect();
+ }
+ }
+
+ protected Database createDataBase(DatabaseMeta databaseMeta) {
+ return new Database(this, databaseMeta);
+ }
+
+ /**
+ * Checks if is active.
+ *
+ * @return true, if is active
+ */
+ public boolean isActive() {
+ return active.get();
+ }
+
+ /**
+ * Stop all activity by setting the stopped property to true.
+ */
+ public void stopAll() {
+ stopped.set(true);
+ }
+
+ /**
+ * Sets the stopped.
+ *
+ * @param stopped the new stopped
+ */
+ public void setStopped(boolean stopped) {
+ this.stopped.set(stopped);
+ }
+
+ /**
+ * Gets the stopped status of this Job.
+ *
+ * @return Returns the stopped status of this Job
+ */
+ public boolean isStopped() {
+ return stopped.get();
+ }
+
+ /**
+ * Gets the start date.
+ *
+ * @return Returns the startDate
+ */
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ /**
+ * Gets the end date.
+ *
+ * @return Returns the endDate
+ */
+ public Date getEndDate() {
+ return endDate;
+ }
+
+ /**
+ * Gets the current date.
+ *
+ * @return Returns the currentDate
+ */
+ public Date getCurrentDate() {
+ return currentDate;
+ }
+
+ /**
+ * Gets the dep date.
+ *
+ * @return Returns the depDate
+ */
+ public Date getDepDate() {
+ return depDate;
+ }
+
+ /**
+ * Gets the log date.
+ *
+ * @return Returns the logDate
+ */
+ public Date getLogDate() {
+ return logDate;
+ }
+
+ /**
+ * Gets the job meta.
+ *
+ * @return Returns the JobMeta
+ */
+ public JobMeta getJobMeta() {
+ return jobMeta;
+ }
+
+ /**
+ * Gets the rep (repository).
+ *
+ * @return Returns the rep
+ */
+ public Repository getRep() {
+ return rep;
+ }
+
+ /**
+ * Gets the thread.
+ *
+ * @return the thread
+ */
+ public Thread getThread() {
+ return this;
+ }
+
+ /**
+ * Gets the job tracker.
+ *
+ * @return Returns the jobTracker
+ */
+ public JobTracker getJobTracker() {
+ return jobTracker;
+ }
+
+ /**
+ * Sets the job tracker.
+ *
+ * @param jobTracker The jobTracker to set
+ */
+ public void setJobTracker(JobTracker jobTracker) {
+ this.jobTracker = jobTracker;
+ }
+
+ /**
+ * Sets the source rows.
+ *
+ * @param sourceRows the new source rows
+ */
+ public void setSourceRows(List sourceRows) {
+ this.sourceRows = sourceRows;
+ }
+
+ /**
+ * Gets the source rows.
+ *
+ * @return the source rows
+ */
+ public List getSourceRows() {
+ return sourceRows;
+ }
+
+ /**
+ * Gets the parent job.
+ *
+ * @return Returns the parentJob
+ */
+ public Job getParentJob() {
+ return parentJob;
+ }
+
+ /**
+ * Sets the parent job.
+ *
+ * @param parentJob The parentJob to set.
+ */
+ public void setParentJob(Job parentJob) {
+ this.logLevel = parentJob.getLogLevel();
+ this.log.setLogLevel(logLevel);
+ this.containerObjectId = log.getContainerObjectId();
+ this.parentJob = parentJob;
+ }
+
+ /**
+ * Gets the result.
+ *
+ * @return the result
+ */
+ public Result getResult() {
+ return result;
+ }
+
+ /**
+ * Sets the result.
+ *
+ * @param result the new result
+ */
+ public void setResult(Result result) {
+ this.result = result;
+ }
+
+ /**
+ * Gets the boolean value of initialized.
+ *
+ * @return Returns the initialized
+ */
+ public boolean isInitialized() {
+ return initialized.get();
+ }
+
+ /**
+ * Gets the batchId.
+ *
+ * @return Returns the batchId
+ */
+ public long getBatchId() {
+ return batchId;
+ }
+
+ /**
+ * Sets the batchId.
+ *
+ * @param batchId The batchId to set
+ */
+ public void setBatchId(long batchId) {
+ this.batchId = batchId;
+ }
+
+ /**
+ * Gets the passedBatchId.
+ *
+ * @return the passedBatchId
+ */
+ public long getPassedBatchId() {
+ return passedBatchId;
+ }
+
+ /**
+ * Sets the passedBatchId.
+ *
+ * @param jobBatchId the jobBatchId to set
+ */
+ public void setPassedBatchId(long jobBatchId) {
+ this.passedBatchId = jobBatchId;
+ }
+
+ /**
+ * Sets the internal kettle variables.
+ *
+ * @param var the new internal kettle variables.
+ */
+ public void setInternalKettleVariables(VariableSpace var) {
+ if (jobMeta != null && jobMeta.getFilename() != null) { // we have a finename that's defined.
+ try {
+ FileObject fileObject = KettleVFS.getFileObject(jobMeta.getFilename(), this);
+ FileName fileName = fileObject.getName();
+
+ // The filename of the transformation
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, fileName.getBaseName());
+
+ // The directory of the transformation
+ FileName fileDir = fileName.getParent();
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, fileDir.getURI());
+ } catch (Exception e) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, "");
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, "");
+ }
+ } else {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, "");
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, "");
+ }
+
+ boolean hasRepoDir = jobMeta.getRepositoryDirectory() != null && jobMeta.getRepository() != null;
+
+ // The name of the job
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_NAME, Const.NVL(jobMeta.getName(), ""));
+
+ // The name of the directory in the repository
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY, hasRepoDir ? jobMeta
+ .getRepositoryDirectory().getPath() : "");
+
+ // setup fallbacks
+ if (hasRepoDir) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, variables.getVariable(
+ Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY));
+ } else {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY, variables.getVariable(
+ Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY));
+ }
+
+ if (hasRepoDir) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, variables.getVariable(
+ Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY));
+ if ("/".equals(variables.getVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY))) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, "");
+ }
+ } else {
+ variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, variables.getVariable(
+ Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY));
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#copyVariablesFrom(org.pentaho.di.core.variables.VariableSpace)
+ */
+ public void copyVariablesFrom(VariableSpace space) {
+ variables.copyVariablesFrom(space);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#environmentSubstitute(java.lang.String)
+ */
+ public String environmentSubstitute(String aString) {
+ return variables.environmentSubstitute(aString);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#environmentSubstitute(java.lang.String[])
+ */
+ public String[] environmentSubstitute(String[] aString) {
+ return variables.environmentSubstitute(aString);
+ }
+
+ public String fieldSubstitute(String aString, RowMetaInterface rowMeta, Object[] rowData)
+ throws KettleValueException {
+ return variables.fieldSubstitute(aString, rowMeta, rowData);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#getParentVariableSpace()
+ */
+ public VariableSpace getParentVariableSpace() {
+ return variables.getParentVariableSpace();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.pentaho.di.core.variables.VariableSpace#setParentVariableSpace(org.pentaho.di.core.variables.VariableSpace)
+ */
+ public void setParentVariableSpace(VariableSpace parent) {
+ variables.setParentVariableSpace(parent);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#getVariable(java.lang.String, java.lang.String)
+ */
+ public String getVariable(String variableName, String defaultValue) {
+ return variables.getVariable(variableName, defaultValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#getVariable(java.lang.String)
+ */
+ public String getVariable(String variableName) {
+ return variables.getVariable(variableName);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#getBooleanValueOfVariable(java.lang.String, boolean)
+ */
+ public boolean getBooleanValueOfVariable(String variableName, boolean defaultValue) {
+ if (!Const.isEmpty(variableName)) {
+ String value = environmentSubstitute(variableName);
+ if (!Const.isEmpty(value)) {
+ return ValueMeta.convertStringToBoolean(value);
+ }
+ }
+ return defaultValue;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.pentaho.di.core.variables.VariableSpace#initializeVariablesFrom(org.pentaho.di.core.variables.VariableSpace)
+ */
+ public void initializeVariablesFrom(VariableSpace parent) {
+ variables.initializeVariablesFrom(parent);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#listVariables()
+ */
+ public String[] listVariables() {
+ return variables.listVariables();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#setVariable(java.lang.String, java.lang.String)
+ */
+ public void setVariable(String variableName, String variableValue) {
+ variables.setVariable(variableName, variableValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#shareVariablesWith(org.pentaho.di.core.variables.VariableSpace)
+ */
+ public void shareVariablesWith(VariableSpace space) {
+ variables = space;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.variables.VariableSpace#injectVariables(java.util.Map)
+ */
+ public void injectVariables(Map prop) {
+ variables.injectVariables(prop);
+ }
+
+ /**
+ * Gets the status.
+ *
+ * @return the status
+ */
+ public String getStatus() {
+ String message;
+
+ if (!initialized.get()) {
+ message = Trans.STRING_WAITING;
+ } else {
+ if (active.get()) {
+ if (stopped.get()) {
+ message = Trans.STRING_HALTING;
+ } else {
+ message = Trans.STRING_RUNNING;
+ }
+ } else {
+ if (stopped.get()) {
+ message = Trans.STRING_STOPPED;
+ } else {
+ message = Trans.STRING_FINISHED;
+ }
+ if (result != null && result.getNrErrors() > 0) {
+ message += " (with errors)";
+ }
+ }
+ }
+
+ return message;
+ }
+
+ /**
+ * Send to slave server.
+ *
+ * @param jobMeta the job meta
+ * @param executionConfiguration the execution configuration
+ * @param repository the repository
+ * @param metaStore the metaStore
+ * @return the string
+ * @throws KettleException the kettle exception
+ */
+ public static String sendToSlaveServer(JobMeta jobMeta, JobExecutionConfiguration executionConfiguration,
+ Repository repository, IMetaStore metaStore) throws KettleException {
+ String carteObjectId;
+ SlaveServer slaveServer = executionConfiguration.getRemoteServer();
+
+ if (slaveServer == null) {
+ throw new KettleException(BaseMessages.getString(PKG, "Job.Log.NoSlaveServerSpecified"));
+ }
+ if (Const.isEmpty(jobMeta.getName())) {
+ throw new KettleException(BaseMessages.getString(PKG, "Job.Log.UniqueJobName"));
+ }
+
+ // Align logging levels between execution configuration and remote server
+ slaveServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel());
+
+ try {
+ // Inject certain internal variables to make it more intuitive.
+ //
+ for (String var : Const.INTERNAL_TRANS_VARIABLES) {
+ executionConfiguration.getVariables().put(var, jobMeta.getVariable(var));
+ }
+ for (String var : Const.INTERNAL_JOB_VARIABLES) {
+ executionConfiguration.getVariables().put(var, jobMeta.getVariable(var));
+ }
+
+ Map jobParams = new HashMap();
+ for (String key : jobMeta.listParameters()) {
+ String value = jobMeta.getParameterValue(key);
+ String defaultValue = jobMeta.getParameterDefault(key);
+ jobParams.put(key,
+ executionConfiguration.getVariables().getOrDefault(key, value == null ? defaultValue : value));
+ }
+
+ executionConfiguration.getParams().putAll(jobParams);
+
+ if (executionConfiguration.isPassingExport()) {
+ // First export the job... slaveServer.getVariable("MASTER_HOST")
+ //
+ FileObject tempFile =
+ KettleVFS.createTempFile("jobExport", ".zip", System.getProperty("java.io.tmpdir"), jobMeta);
+
+ TopLevelResource topLevelResource =
+ ResourceUtil.serializeResourceExportInterface(tempFile.getName().toString(), jobMeta, jobMeta, repository,
+ metaStore, executionConfiguration.getXML(), CONFIGURATION_IN_EXPORT_FILENAME);
+
+ // Send the zip file over to the slave server...
+ //
+ String result =
+ slaveServer.sendExport(topLevelResource.getArchiveName(), AddExportServlet.TYPE_JOB, topLevelResource
+ .getBaseResourceName());
+ WebResult webResult = WebResult.fromXMLString(result);
+ if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
+ throw new KettleException("There was an error passing the exported job to the remote server: " + Const.CR
+ + webResult.getMessage());
+ }
+ carteObjectId = webResult.getId();
+ } else {
+ String xml = new JobConfiguration(jobMeta, executionConfiguration).getXML();
+
+ String reply = slaveServer.sendXML(xml, RegisterJobServlet.CONTEXT_PATH + "/?xml=Y");
+ WebResult webResult = WebResult.fromXMLString(reply);
+ if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
+ throw new KettleException("There was an error posting the job on the remote server: " + Const.CR + webResult
+ .getMessage());
+ }
+ carteObjectId = webResult.getId();
+ }
+
+ // Start the job
+ //
+ String reply =
+ slaveServer.execService(StartJobServlet.CONTEXT_PATH + "/?name=" + URLEncoder.encode(jobMeta.getName(),
+ "UTF-8") + "&xml=Y&id=" + carteObjectId);
+ WebResult webResult = WebResult.fromXMLString(reply);
+ if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
+ throw new KettleException("There was an error starting the job on the remote server: " + Const.CR + webResult
+ .getMessage());
+ }
+ return carteObjectId;
+ } catch (KettleException ke) {
+ throw ke;
+ } catch (Exception e) {
+ throw new KettleException(e);
+ }
+ }
+
+ /**
+ * Add a job listener to the job
+ *
+ * @param jobListener the job listener to add
+ */
+ public void addJobListener(JobListener jobListener) {
+ synchronized (jobListeners) {
+ jobListeners.add(jobListener);
+ }
+ }
+
+ /**
+ * Adds the job entry listener.
+ *
+ * @param jobEntryListener the job entry listener
+ */
+ public void addJobEntryListener(JobEntryListener jobEntryListener) {
+ jobEntryListeners.add(jobEntryListener);
+ }
+
+ /**
+ * Remove a job listener from the job
+ *
+ * @param jobListener the job listener to remove
+ */
+ public void removeJobListener(JobListener jobListener) {
+ synchronized (jobListeners) {
+ jobListeners.remove(jobListener);
+ }
+ }
+
+ /**
+ * Remove a job entry listener from the job
+ *
+ * @param jobEntryListener the job entry listener to remove
+ */
+ public void removeJobEntryListener(JobEntryListener jobEntryListener) {
+ jobEntryListeners.remove(jobEntryListener);
+ }
+
+ /**
+ * Gets the job entry listeners.
+ *
+ * @return the job entry listeners
+ */
+ public List getJobEntryListeners() {
+ return jobEntryListeners;
+ }
+
+ /**
+ * Gets the job listeners.
+ *
+ * @return the job listeners
+ */
+ public List getJobListeners() {
+ synchronized (jobListeners) {
+ return new ArrayList(jobListeners);
+ }
+ }
+
+ /**
+ * Gets the boolean value of finished.
+ *
+ * @return the finished
+ */
+ public boolean isFinished() {
+ return finished.get();
+ }
+
+ /**
+ * Sets the value of finished.
+ *
+ * @param finished the finished to set
+ */
+ public void setFinished(boolean finished) {
+ this.finished.set(finished);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#addParameterDefinition(java.lang.String, java.lang.String,
+ * java.lang.String)
+ */
+ public void addParameterDefinition(String key, String defValue, String description) throws DuplicateParamException {
+ namedParams.addParameterDefinition(key, defValue, description);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#getParameterDescription(java.lang.String)
+ */
+ public String getParameterDescription(String key) throws UnknownParamException {
+ return namedParams.getParameterDescription(key);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#getParameterDefault(java.lang.String)
+ */
+ public String getParameterDefault(String key) throws UnknownParamException {
+ return namedParams.getParameterDefault(key);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#getParameterValue(java.lang.String)
+ */
+ public String getParameterValue(String key) throws UnknownParamException {
+ return namedParams.getParameterValue(key);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#listParameters()
+ */
+ public String[] listParameters() {
+ return namedParams.listParameters();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#setParameterValue(java.lang.String, java.lang.String)
+ */
+ public void setParameterValue(String key, String value) throws UnknownParamException {
+ namedParams.setParameterValue(key, value);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#eraseParameters()
+ */
+ public void eraseParameters() {
+ namedParams.eraseParameters();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#clearParameters()
+ */
+ public void clearParameters() {
+ namedParams.clearParameters();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#activateParameters()
+ */
+ public void activateParameters() {
+ String[] keys = listParameters();
+
+ for (String key : keys) {
+ String value;
+ try {
+ value = getParameterValue(key);
+ } catch (UnknownParamException e) {
+ value = "";
+ }
+ String defValue;
+ try {
+ defValue = getParameterDefault(key);
+ } catch (UnknownParamException e) {
+ defValue = "";
+ }
+
+ if (Const.isEmpty(value)) {
+ setVariable(key, Const.NVL(defValue, ""));
+ } else {
+ setVariable(key, Const.NVL(value, ""));
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.parameters.NamedParams#copyParametersFrom(org.pentaho.di.core.parameters.NamedParams)
+ */
+ public void copyParametersFrom(NamedParams params) {
+ namedParams.copyParametersFrom(params);
+ }
+
+ /**
+ * Sets the socket repository.
+ *
+ * @param socketRepository the new socket repository
+ */
+ public void setSocketRepository(SocketRepository socketRepository) {
+ this.socketRepository = socketRepository;
+ }
+
+ /**
+ * Gets the socket repository.
+ *
+ * @return the socket repository
+ */
+ public SocketRepository getSocketRepository() {
+ return socketRepository;
+ }
+
+ /**
+ * Gets the log channel interface.
+ *
+ * @return LogChannelInterface
+ */
+ public LogChannelInterface getLogChannel() {
+ return log;
+ }
+
+ /**
+ * Gets the job name.
+ *
+ * @return jobName
+ */
+ public String getObjectName() {
+ return getJobname();
+ }
+
+ /**
+ * Always returns null for Job.
+ *
+ * @return null
+ */
+ public String getObjectCopy() {
+ return null;
+ }
+
+ /**
+ * Gets the file name.
+ *
+ * @return the filename
+ */
+ public String getFilename() {
+ if (jobMeta == null) {
+ return null;
+ }
+ return jobMeta.getFilename();
+ }
+
+ /**
+ * Gets the log channel id.
+ *
+ * @return the logChannelId
+ */
+ public String getLogChannelId() {
+ return log.getLogChannelId();
+ }
+
+ /**
+ * Gets the jobMeta's object id.
+ *
+ * @return ObjectId
+ */
+ public ObjectId getObjectId() {
+ if (jobMeta == null) {
+ return null;
+ }
+ return jobMeta.getObjectId();
+ }
+
+ /**
+ * Gets the job meta's object revision.
+ *
+ * @return ObjectRevision
+ */
+ public ObjectRevision getObjectRevision() {
+ if (jobMeta == null) {
+ return null;
+ }
+ return jobMeta.getObjectRevision();
+ }
+
+ /**
+ * Gets LoggingObjectType.JOB, which is always the value for Job.
+ *
+ * @return LoggingObjectType LoggingObjectType.JOB
+ */
+ public LoggingObjectType getObjectType() {
+ return LoggingObjectType.JOB;
+ }
+
+ /**
+ * Gets parent logging object.
+ *
+ * @return parentLoggingObject
+ */
+ public LoggingObjectInterface getParent() {
+ return parentLoggingObject;
+ }
+
+ /**
+ * Gets the job meta's repository directory interface.
+ *
+ * @return RepositoryDirectoryInterface
+ */
+ public RepositoryDirectoryInterface getRepositoryDirectory() {
+ if (jobMeta == null) {
+ return null;
+ }
+ return jobMeta.getRepositoryDirectory();
+ }
+
+ /**
+ * Gets the logLevel.
+ *
+ * @return logLevel
+ */
+ public LogLevel getLogLevel() {
+ return logLevel;
+ }
+
+ /**
+ * Sets the log level.
+ *
+ * @param logLevel the new log level
+ */
+ public void setLogLevel(LogLevel logLevel) {
+ this.logLevel = logLevel;
+ log.setLogLevel(logLevel);
+ }
+
+ /**
+ * Gets the logging hierarchy.
+ *
+ * @return the logging hierarchy
+ */
+ public List getLoggingHierarchy() {
+ List hierarchy = new ArrayList();
+ List childIds = LoggingRegistry.getInstance().getLogChannelChildren(getLogChannelId());
+ for (String childId : childIds) {
+ LoggingObjectInterface loggingObject = LoggingRegistry.getInstance().getLoggingObject(childId);
+ if (loggingObject != null) {
+ hierarchy.add(new LoggingHierarchy(getLogChannelId(), batchId, loggingObject));
+ }
+ }
+
+ return hierarchy;
+ }
+
+ /**
+ * Gets the boolean value of interactive.
+ *
+ * @return the interactive
+ */
+ public boolean isInteractive() {
+ return interactive;
+ }
+
+ /**
+ * Sets the value of interactive.
+ *
+ * @param interactive the interactive to set
+ */
+ public void setInteractive(boolean interactive) {
+ this.interactive = interactive;
+ }
+
+ /**
+ * Gets the activeJobEntryTransformations.
+ *
+ * @return the activeJobEntryTransformations
+ */
+ public Map getActiveJobEntryTransformations() {
+ return activeJobEntryTransformations;
+ }
+
+ /**
+ * Gets the activeJobEntryJobs.
+ *
+ * @return the activeJobEntryJobs
+ */
+ public Map getActiveJobEntryJobs() {
+ return activeJobEntryJobs;
+ }
+
+ /**
+ * Gets a flat list of results in THIS job, in the order of execution of job entries.
+ *
+ * @return A flat list of results in THIS job, in the order of execution of job entries
+ */
+ public List getJobEntryResults() {
+ synchronized (jobEntryResults) {
+ return new ArrayList(jobEntryResults);
+ }
+ }
+
+ /**
+ * Gets the carteObjectId.
+ *
+ * @return the carteObjectId
+ */
+ public String getContainerObjectId() {
+ return containerObjectId;
+ }
+
+ /**
+ * Sets the execution container object id (containerObjectId).
+ *
+ * @param containerObjectId the execution container object id to set
+ */
+ public void setContainerObjectId(String containerObjectId) {
+ this.containerObjectId = containerObjectId;
+ }
+
+ /**
+ * Gets the parent logging object.
+ *
+ * @return the parent logging object
+ */
+ public LoggingObjectInterface getParentLoggingObject() {
+ return parentLoggingObject;
+ }
+
+ /**
+ * Gets the registration date. For job, this always returns null
+ *
+ * @return null
+ */
+ public Date getRegistrationDate() {
+ return null;
+ }
+
+ /**
+ * Gets the start job entry copy.
+ *
+ * @return the startJobEntryCopy
+ */
+ public JobEntryCopy getStartJobEntryCopy() {
+ return startJobEntryCopy;
+ }
+
+ /**
+ * Sets the start job entry copy.
+ *
+ * @param startJobEntryCopy the startJobEntryCopy to set
+ */
+ public void setStartJobEntryCopy(JobEntryCopy startJobEntryCopy) {
+ this.startJobEntryCopy = startJobEntryCopy;
+ }
+
+ /**
+ * Gets the executing server.
+ *
+ * @return the executingServer
+ */
+ public String getExecutingServer() {
+ return executingServer;
+ }
+
+ /**
+ * Sets the executing server.
+ *
+ * @param executingServer the executingServer to set
+ */
+ public void setExecutingServer(String executingServer) {
+ this.executingServer = executingServer;
+ }
+
+ /**
+ * Gets the executing user.
+ *
+ * @return the executingUser
+ */
+ public String getExecutingUser() {
+ return executingUser;
+ }
+
+ /**
+ * Sets the executing user.
+ *
+ * @param executingUser the executingUser to set
+ */
+ public void setExecutingUser(String executingUser) {
+ this.executingUser = executingUser;
+ }
+
+ @Override
+ public boolean isGatheringMetrics() {
+ return log != null && log.isGatheringMetrics();
+ }
+
+ @Override
+ public void setGatheringMetrics(boolean gatheringMetrics) {
+ if (log != null) {
+ log.setGatheringMetrics(gatheringMetrics);
+ }
+ }
+
+ @Override
+ public boolean isForcingSeparateLogging() {
+ return log != null && log.isForcingSeparateLogging();
+ }
+
+ @Override
+ public void setForcingSeparateLogging(boolean forcingSeparateLogging) {
+ if (log != null) {
+ log.setForcingSeparateLogging(forcingSeparateLogging);
+ }
+ }
+
+ /**
+ * Gets the transaction id.
+ *
+ * @return the transactionId
+ */
+ public String getTransactionId() {
+ return transactionId;
+ }
+
+ /**
+ * Sets the transaction id.
+ *
+ * @param transactionId the transactionId to set
+ */
+ public void setTransactionId(String transactionId) {
+ this.transactionId = transactionId;
+ }
+
+ public List getDelegationListeners() {
+ return delegationListeners;
+ }
+
+ public void setDelegationListeners(List delegationListeners) {
+ this.delegationListeners = delegationListeners;
+ }
+
+ public void addDelegationListener(DelegationListener delegationListener) {
+ delegationListeners.add(delegationListener);
+ }
+
+ public String[] getArguments() {
+ return arguments;
+ }
+
+ public void setArguments(String[] arguments) {
+ this.arguments = arguments;
+ }
+
+ public Trans getParentTrans() {
+ return parentTrans;
+ }
+
+ public void setParentTrans(Trans parentTrans) {
+ this.parentTrans = parentTrans;
+ }
+
+ public Map getExtensionDataMap() {
+ return extensionDataMap;
+ }
+
+ public Result getStartJobEntryResult() {
+ return startJobEntryResult;
+ }
+
+ public void setStartJobEntryResult(Result startJobEntryResult) {
+ this.startJobEntryResult = startJobEntryResult;
+ }
+
+ protected ExecutorService startHeartbeat(final long intervalInSeconds) {
+
+ final ScheduledExecutorService heartbeat = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread thread = new Thread(r, "Job Heartbeat Thread for: " + getName());
+ thread.setDaemon(true);
+ return thread;
+ }
+ });
+
+ heartbeat.scheduleAtFixedRate(new Runnable() {
+ public void run() {
+
+ if (Job.this.isFinished()) {
+ log.logBasic("Shutting down heartbeat signal for " + jobMeta.getName());
+ shutdownHeartbeat(heartbeat);
+ return;
+ }
+
+ try {
+
+ log.logDebug("Triggering heartbeat signal for " + jobMeta.getName() + " at every " + intervalInSeconds
+ + " seconds");
+ ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.JobHeartbeat.id, Job.this);
+
+ } catch (KettleException e) {
+ log.logError(e.getMessage(), e);
+ }
+ }
+ }, intervalInSeconds /* initial delay */, intervalInSeconds /* interval delay */, TimeUnit.SECONDS);
+
+ return heartbeat;
+ }
+
+ protected void shutdownHeartbeat(ExecutorService heartbeat) {
+
+ if (heartbeat != null) {
+
+ try {
+ heartbeat.shutdownNow(); // prevents waiting tasks from starting and attempts to stop currently executing ones
+
+ } catch (Throwable t) {
+ /* do nothing */
+ }
+ }
+ }
+
+ private int getHeartbeatIntervalInSeconds() {
+
+ JobMeta meta = this.jobMeta;
+
+ // 1 - check if there's a user defined value ( job-specific ) heartbeat periodic interval;
+ // 2 - check if there's a default defined value ( job-specific ) heartbeat periodic interval;
+ // 3 - use default Const.HEARTBEAT_PERIODIC_INTERVAL_IN_SECS if none of the above have been set
+
+ try {
+
+ if (meta != null) {
+
+ return Const.toInt(meta.getParameterValue(Const.VARIABLE_HEARTBEAT_PERIODIC_INTERVAL_SECS), Const.toInt(meta
+ .getParameterDefault(Const.VARIABLE_HEARTBEAT_PERIODIC_INTERVAL_SECS),
+ Const.HEARTBEAT_PERIODIC_INTERVAL_IN_SECS));
+ }
+
+ } catch (Exception e) {
+ /* do nothing, return Const.HEARTBEAT_PERIODIC_INTERVAL_IN_SECS */
+ }
+
+ return Const.HEARTBEAT_PERIODIC_INTERVAL_IN_SECS;
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/job/JobMeta.java b/pentaho-kettle/src/main/java/org/pentaho/di/job/JobMeta.java
new file mode 100644
index 0000000..f3ebc75
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/job/JobMeta.java
@@ -0,0 +1,2823 @@
+// CHECKSTYLE:FileLength:OFF
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.job;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.pentaho.di.base.AbstractMeta;
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.*;
+import org.pentaho.di.core.attributes.AttributesUtil;
+import org.pentaho.di.core.database.Database;
+import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.exception.*;
+import org.pentaho.di.core.extension.ExtensionPointHandler;
+import org.pentaho.di.core.extension.KettleExtensionPoint;
+import org.pentaho.di.core.gui.OverwritePrompter;
+import org.pentaho.di.core.gui.Point;
+import org.pentaho.di.core.logging.*;
+import org.pentaho.di.core.logging.LogTablePluginInterface.TableType;
+import org.pentaho.di.core.parameters.NamedParamsDefault;
+import org.pentaho.di.core.parameters.UnknownParamException;
+import org.pentaho.di.core.plugins.PluginInterface;
+import org.pentaho.di.core.plugins.PluginRegistry;
+import org.pentaho.di.core.reflection.StringSearchResult;
+import org.pentaho.di.core.reflection.StringSearcher;
+import org.pentaho.di.core.row.RowMetaInterface;
+import org.pentaho.di.core.util.StringUtil;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.core.xml.XMLHandler;
+import org.pentaho.di.core.xml.XMLInterface;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.entries.missing.MissingEntry;
+import org.pentaho.di.job.entries.special.JobEntrySpecial;
+import org.pentaho.di.job.entry.JobEntryCopy;
+import org.pentaho.di.job.entry.JobEntryInterface;
+import org.pentaho.di.repository.*;
+import org.pentaho.di.resource.ResourceDefinition;
+import org.pentaho.di.resource.ResourceExportInterface;
+import org.pentaho.di.resource.ResourceNamingInterface;
+import org.pentaho.di.resource.ResourceReference;
+import org.pentaho.di.shared.SharedObjectInterface;
+import org.pentaho.di.shared.SharedObjects;
+import org.pentaho.metastore.api.IMetaStore;
+import org.pentaho.metastore.api.exceptions.MetaStoreException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+
+import java.io.InputStream;
+import java.util.*;
+
+/**
+ * The definition of a PDI job is represented by a JobMeta object. It is typically loaded from a .kjb file, a PDI
+ * repository, or it is generated dynamically. The declared parameters of the job definition are then queried using
+ * listParameters() and assigned values using calls to setParameterValue(..). JobMeta provides methods to load, save,
+ * verify, etc.
+ *
+ * @author Matt
+ * @since 11-08-2003
+ */
+public class JobMeta extends AbstractMeta
+ implements Cloneable, Comparable, XMLInterface, ResourceExportInterface, RepositoryElementInterface,
+ LoggingObjectInterface {
+
+ private static Class PKG = JobMeta.class; // for i18n purposes, needed by Translator2!!
+
+ public static final String XML_TAG = "job";
+
+ protected static final String XML_TAG_SLAVESERVERS = "slaveservers";
+
+ /**
+ * A constant specifying the repository element type as a Job.
+ */
+ public static final RepositoryObjectType REPOSITORY_ELEMENT_TYPE = RepositoryObjectType.JOB;
+
+ protected String jobVersion;
+
+ protected int jobStatus;
+
+ protected List jobcopies;
+
+ protected List jobhops;
+
+ protected String[] arguments;
+
+ protected boolean changedEntries, changedHops;
+
+ protected JobLogTable jobLogTable;
+
+ protected JobEntryLogTable jobEntryLogTable;
+
+ protected List extraLogTables;
+
+ /**
+ * Constant = "SPECIAL"
+ **/
+ public static final String STRING_SPECIAL = "SPECIAL";
+
+ /**
+ * Constant = "START"
+ **/
+ public static final String STRING_SPECIAL_START = "START";
+
+ /**
+ * Constant = "DUMMY"
+ **/
+ public static final String STRING_SPECIAL_DUMMY = "DUMMY";
+
+ /**
+ * Constant = "OK"
+ **/
+ public static final String STRING_SPECIAL_OK = "OK";
+
+ /**
+ * Constant = "ERROR"
+ **/
+ public static final String STRING_SPECIAL_ERROR = "ERROR";
+
+ /**
+ * The loop cache.
+ */
+ protected Map loopCache;
+
+ /**
+ * List of booleans indicating whether or not to remember the size and position of the different windows...
+ */
+ public boolean[] max = new boolean[1];
+
+ protected boolean batchIdPassed;
+
+ protected static final String XML_TAG_PARAMETERS = "parameters";
+
+ private List missingEntries;
+
+ /**
+ * Instantiates a new job meta.
+ */
+ public JobMeta() {
+ clear();
+ initializeVariablesFrom(null);
+ }
+
+ /**
+ * Clears or reinitializes many of the JobMeta properties.
+ */
+ @Override
+ public void clear() {
+ jobcopies = new ArrayList();
+ jobhops = new ArrayList();
+
+ jobLogTable = JobLogTable.getDefault(this, this);
+ jobEntryLogTable = JobEntryLogTable.getDefault(this, this);
+ extraLogTables = new ArrayList();
+
+ List plugins = PluginRegistry.getInstance().getPlugins(LogTablePluginType.class);
+ for (PluginInterface plugin : plugins) {
+ try {
+ LogTablePluginInterface logTablePluginInterface = (LogTablePluginInterface) PluginRegistry.getInstance()
+ .loadClass(plugin);
+ if (logTablePluginInterface.getType() == TableType.JOB) {
+ logTablePluginInterface.setContext(this, this);
+ extraLogTables.add(logTablePluginInterface);
+ }
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError("Error loading log table plugin with ID " + plugin.getIds()[0], e);
+ }
+ }
+
+ arguments = null;
+
+ super.clear();
+ loopCache = new HashMap();
+ addDefaults();
+ jobStatus = -1;
+ jobVersion = null;
+
+ // setInternalKettleVariables(); Don't clear the internal variables for
+ // ad-hoc jobs, it's ruins the previews
+ // etc.
+ }
+
+ /**
+ * Adds the defaults.
+ */
+ public void addDefaults() {
+ /*
+ * addStart(); // Add starting point! addDummy(); // Add dummy! addOK(); // errors == 0 evaluation addError(); //
+ * errors != 0 evaluation
+ */
+
+ clearChanged();
+ }
+
+ /**
+ * Creates the start entry.
+ *
+ * @return the job entry copy
+ */
+ public static final JobEntryCopy createStartEntry() {
+ JobEntrySpecial jobEntrySpecial = new JobEntrySpecial(STRING_SPECIAL_START, true, false);
+ JobEntryCopy jobEntry = new JobEntryCopy();
+ jobEntry.setObjectId(null);
+ jobEntry.setEntry(jobEntrySpecial);
+ jobEntry.setLocation(50, 50);
+ jobEntry.setDrawn(false);
+ jobEntry.setDescription(BaseMessages.getString(PKG, "JobMeta.StartJobEntry.Description"));
+ return jobEntry;
+
+ }
+
+ /**
+ * Creates the dummy entry.
+ *
+ * @return the job entry copy
+ */
+ public static final JobEntryCopy createDummyEntry() {
+ JobEntrySpecial jobEntrySpecial = new JobEntrySpecial(STRING_SPECIAL_DUMMY, false, true);
+ JobEntryCopy jobEntry = new JobEntryCopy();
+ jobEntry.setObjectId(null);
+ jobEntry.setEntry(jobEntrySpecial);
+ jobEntry.setLocation(50, 50);
+ jobEntry.setDrawn(false);
+ jobEntry.setDescription(BaseMessages.getString(PKG, "JobMeta.DummyJobEntry.Description"));
+ return jobEntry;
+ }
+
+ /**
+ * Gets the start.
+ *
+ * @return the start
+ */
+ public JobEntryCopy getStart() {
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy cge = getJobEntry(i);
+ if (cge.isStart()) {
+ return cge;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets the dummy.
+ *
+ * @return the dummy
+ */
+ public JobEntryCopy getDummy() {
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy cge = getJobEntry(i);
+ if (cge.isDummy()) {
+ return cge;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Compares two transformation on name, filename
+ */
+ public int compare(JobMeta j1, JobMeta j2) {
+ if (Const.isEmpty(j1.getName()) && !Const.isEmpty(j2.getName())) {
+ return -1;
+ }
+ if (!Const.isEmpty(j1.getName()) && Const.isEmpty(j2.getName())) {
+ return 1;
+ }
+ if (Const.isEmpty(j1.getName()) && Const.isEmpty(j2.getName()) || j1.getName().equals(j2.getName())) {
+ if (Const.isEmpty(j1.getFilename()) && !Const.isEmpty(j2.getFilename())) {
+ return -1;
+ }
+ if (!Const.isEmpty(j1.getFilename()) && Const.isEmpty(j2.getFilename())) {
+ return 1;
+ }
+ if (Const.isEmpty(j1.getFilename()) && Const.isEmpty(j2.getFilename())) {
+ return 0;
+ }
+ return j1.getFilename().compareTo(j2.getFilename());
+ }
+
+ // Compare by name : repositories etc.
+ //
+ if (j1.getObjectRevision() != null && j2.getObjectRevision() == null) {
+ return 1;
+ }
+ if (j1.getObjectRevision() == null && j2.getObjectRevision() != null) {
+ return -1;
+ }
+ int cmp;
+ if (j1.getObjectRevision() == null && j2.getObjectRevision() == null) {
+ cmp = 0;
+ } else {
+ cmp = j1.getObjectRevision().getName().compareTo(j2.getObjectRevision().getName());
+ }
+ if (cmp == 0) {
+ return j1.getName().compareTo(j2.getName());
+ } else {
+ return cmp;
+ }
+ }
+
+ /**
+ * Compares this job's meta-data to the specified job's meta-data. This method simply calls compare(this, o)
+ *
+ * @param o the o
+ * @return the int
+ * @see #compare(JobMeta, JobMeta)
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ public int compareTo(JobMeta o) {
+ return compare(this, o);
+ }
+
+ /**
+ * Checks whether this job's meta-data object is equal to the specified object. If the specified object is not an
+ * instance of JobMeta, false is returned. Otherwise the method returns whether a call to compare() indicates equality
+ * (i.e. compare(this, (JobMeta)obj)==0).
+ *
+ * @param obj the obj
+ * @return true, if successful
+ * @see #compare(JobMeta, JobMeta)
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ public boolean equals(Object obj) {
+ if (!(obj instanceof JobMeta)) {
+ return false;
+ }
+
+ return compare(this, (JobMeta) obj) == 0;
+ }
+
+ /**
+ * Clones the job meta-data object.
+ *
+ * @return a clone of the job meta-data object
+ * @see java.lang.Object#clone()
+ */
+ public Object clone() {
+ return realClone(true);
+ }
+
+ /**
+ * Perform a real clone of the job meta-data object, including cloning all lists and copying all values. If the
+ * doClear parameter is true, the clone will be cleared of ALL values before the copy. If false, only the copied
+ * fields will be cleared.
+ *
+ * @param doClear Whether to clear all of the clone's data before copying from the source object
+ * @return a real clone of the calling object
+ */
+ public Object realClone(boolean doClear) {
+ try {
+ JobMeta jobMeta = (JobMeta) super.clone();
+ if (doClear) {
+ jobMeta.clear();
+ } else {
+ jobMeta.jobcopies = new ArrayList();
+ jobMeta.jobhops = new ArrayList();
+ jobMeta.notes = new ArrayList();
+ jobMeta.databases = new ArrayList();
+ jobMeta.slaveServers = new ArrayList();
+ jobMeta.namedParams = new NamedParamsDefault();
+ }
+
+ for (JobEntryCopy entry : jobcopies) {
+ jobMeta.jobcopies.add((JobEntryCopy) entry.clone_deep());
+ }
+ for (JobHopMeta entry : jobhops) {
+ jobMeta.jobhops.add((JobHopMeta) entry.clone());
+ }
+ for (NotePadMeta entry : notes) {
+ jobMeta.notes.add((NotePadMeta) entry.clone());
+ }
+ for (DatabaseMeta entry : databases) {
+ jobMeta.databases.add((DatabaseMeta) entry.clone());
+ }
+ for (SlaveServer slave : slaveServers) {
+ jobMeta.getSlaveServers().add((SlaveServer) slave.clone());
+ }
+ for (String key : listParameters()) {
+ jobMeta.addParameterDefinition(key, getParameterDefault(key), getParameterDescription(key));
+ }
+ return jobMeta;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Gets the job log table.
+ *
+ * @return the job log table
+ */
+ public JobLogTable getJobLogTable() {
+ return jobLogTable;
+ }
+
+ /**
+ * Sets the job log table.
+ *
+ * @param jobLogTable the new job log table
+ */
+ public void setJobLogTable(JobLogTable jobLogTable) {
+ this.jobLogTable = jobLogTable;
+ }
+
+ /**
+ * Clears the different changed flags of the job.
+ */
+ @Override
+ public void clearChanged() {
+ changedEntries = false;
+ changedHops = false;
+
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy entry = getJobEntry(i);
+ entry.setChanged(false);
+ }
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+ hi.setChanged(false);
+ }
+ super.clearChanged();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.changed.ChangedFlag#hasChanged()
+ */
+ @Override
+ public boolean hasChanged() {
+ if (super.hasChanged()) {
+ return true;
+ }
+
+ if (haveJobEntriesChanged()) {
+ return true;
+ }
+ if (haveJobHopsChanged()) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private Set getUsedDatabaseMetas() {
+ Set databaseMetas = new HashSet();
+ for (JobEntryCopy jobEntryCopy : getJobCopies()) {
+ DatabaseMeta[] dbs = jobEntryCopy.getEntry().getUsedDatabaseConnections();
+ if (dbs != null) {
+ for (DatabaseMeta db : dbs) {
+ databaseMetas.add(db);
+ }
+ }
+ }
+
+ databaseMetas.add(jobLogTable.getDatabaseMeta());
+
+ for (LogTableInterface logTable : getExtraLogTables()) {
+ databaseMetas.add(logTable.getDatabaseMeta());
+ }
+ return databaseMetas;
+ }
+
+ /**
+ * This method asks all steps in the transformation whether or not the specified database connection is used. The
+ * connection is used in the transformation if any of the steps uses it or if it is being used to log to.
+ *
+ * @param databaseMeta The connection to check
+ * @return true if the connection is used in this transformation.
+ */
+ public boolean isDatabaseConnectionUsed(DatabaseMeta databaseMeta) {
+ return getUsedDatabaseMetas().contains(databaseMeta);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.EngineMetaInterface#getFileType()
+ */
+ public String getFileType() {
+ return LastUsedFile.FILE_TYPE_JOB;
+ }
+
+ /**
+ * Gets the job filter names.
+ *
+ * @return the filter names
+ * @see org.pentaho.di.core.EngineMetaInterface#getFilterNames()
+ */
+ public String[] getFilterNames() {
+ return Const.getJobFilterNames();
+ }
+
+ /**
+ * Gets the job filter extensions. For JobMeta, this method returns the value of {@link Const.STRING_JOB_FILTER_EXT}
+ *
+ * @return the filter extensions
+ * @see org.pentaho.di.core.EngineMetaInterface#getFilterExtensions()
+ */
+ public String[] getFilterExtensions() {
+ return Const.STRING_JOB_FILTER_EXT;
+ }
+
+ /**
+ * Gets the default extension for a job. For JobMeta, this method returns the value of
+ * {@link Const#STRING_JOB_DEFAULT_EXT}
+ *
+ * @return the default extension
+ * @see org.pentaho.di.core.EngineMetaInterface#getDefaultExtension()
+ */
+ public String getDefaultExtension() {
+ return Const.STRING_JOB_DEFAULT_EXT;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.xml.XMLInterface#getXML()
+ */
+ public String getXML() {
+ Props props = null;
+ if (Props.isInitialized()) {
+ props = Props.getInstance();
+ }
+
+ StringBuffer retval = new StringBuffer(500);
+
+ retval.append("<").append(XML_TAG).append(">").append(Const.CR);
+
+ retval.append(" ").append(XMLHandler.addTagValue("name", getName()));
+
+ retval.append(" ").append(XMLHandler.addTagValue("description", description));
+ retval.append(" ").append(XMLHandler.addTagValue("extended_description", extendedDescription));
+ retval.append(" ").append(XMLHandler.addTagValue("job_version", jobVersion));
+ if (jobStatus >= 0) {
+ retval.append(" ").append(XMLHandler.addTagValue("job_status", jobStatus));
+ }
+
+ retval.append(" ").append(XMLHandler.addTagValue("directory",
+ (directory != null ? directory.getPath() : RepositoryDirectory.DIRECTORY_SEPARATOR)));
+ retval.append(" ").append(XMLHandler.addTagValue("created_user", createdUser));
+ retval.append(" ").append(XMLHandler.addTagValue("created_date", XMLHandler.date2string(createdDate)));
+ retval.append(" ").append(XMLHandler.addTagValue("modified_user", modifiedUser));
+ retval.append(" ").append(XMLHandler.addTagValue("modified_date", XMLHandler.date2string(modifiedDate)));
+
+ retval.append(" ").append(XMLHandler.openTag(XML_TAG_PARAMETERS)).append(Const.CR);
+ String[] parameters = listParameters();
+ for (int idx = 0; idx < parameters.length; idx++) {
+ retval.append(" ").append(XMLHandler.openTag("parameter")).append(Const.CR);
+ retval.append(" ").append(XMLHandler.addTagValue("name", parameters[idx]));
+ try {
+ retval.append(" ")
+ .append(XMLHandler.addTagValue("default_value", getParameterDefault(parameters[idx])));
+ retval.append(" ")
+ .append(XMLHandler.addTagValue("description", getParameterDescription(parameters[idx])));
+ } catch (UnknownParamException e) {
+ // skip the default value and/or description. This exception should never happen because we use listParameters()
+ // above.
+ }
+ retval.append(" ").append(XMLHandler.closeTag("parameter")).append(Const.CR);
+ }
+ retval.append(" ").append(XMLHandler.closeTag(XML_TAG_PARAMETERS)).append(Const.CR);
+
+ Set usedDatabaseMetas = getUsedDatabaseMetas();
+ // Save the database connections...
+ for (int i = 0; i < nrDatabases(); i++) {
+ DatabaseMeta dbMeta = getDatabase(i);
+ if (props != null && props.areOnlyUsedConnectionsSavedToXML()) {
+ if (usedDatabaseMetas.contains(dbMeta)) {
+ retval.append(dbMeta.getXML());
+ }
+ } else {
+ retval.append(dbMeta.getXML());
+ }
+ }
+
+ // The slave servers...
+ //
+ retval.append(" ").append(XMLHandler.openTag(XML_TAG_SLAVESERVERS)).append(Const.CR);
+ for (int i = 0; i < slaveServers.size(); i++) {
+ SlaveServer slaveServer = slaveServers.get(i);
+ retval.append(" ").append(slaveServer.getXML()).append(Const.CR);
+ }
+ retval.append(" ").append(XMLHandler.closeTag(XML_TAG_SLAVESERVERS)).append(Const.CR);
+
+ // Append the job logging information...
+ //
+ for (LogTableInterface logTable : getLogTables()) {
+ retval.append(logTable.getXML());
+ }
+
+ retval.append(" ").append(XMLHandler.addTagValue("pass_batchid", batchIdPassed));
+ retval.append(" ").append(XMLHandler.addTagValue("shared_objects_file", sharedObjectsFile));
+
+ retval.append(" ").append(Const.CR);
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy jge = getJobEntry(i);
+ jge.getEntry().setRepository(repository);
+ retval.append(jge.getXML());
+ }
+ retval.append(" ").append(Const.CR);
+
+ retval.append(" ").append(Const.CR);
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+ retval.append(hi.getXML());
+ }
+ retval.append(" ").append(Const.CR);
+
+ retval.append(" ").append(Const.CR);
+ for (int i = 0; i < nrNotes(); i++) {
+ NotePadMeta ni = getNote(i);
+ retval.append(ni.getXML());
+ }
+ retval.append(" ").append(Const.CR);
+
+ // Also store the attribute groups
+ //
+ retval.append(AttributesUtil.getAttributesXml(attributesMap)).append(Const.CR);
+
+ retval.append("").append(Const.CR);
+
+ return retval.toString();
+ }
+
+ /**
+ * Instantiates a new job meta.
+ *
+ * @param fname the fname
+ * @param rep the rep
+ * @throws KettleXMLException the kettle xml exception
+ */
+ public JobMeta(String fname, Repository rep) throws KettleXMLException {
+ this(null, fname, rep, null);
+ }
+
+ /**
+ * Instantiates a new job meta.
+ *
+ * @param fname the fname
+ * @param rep the rep
+ * @param prompter the prompter
+ * @throws KettleXMLException the kettle xml exception
+ */
+ public JobMeta(String fname, Repository rep, OverwritePrompter prompter) throws KettleXMLException {
+ this(null, fname, rep, prompter);
+ }
+
+ /**
+ * Load the job from the XML file specified.
+ *
+ * @param fname The filename to load as a job
+ * @param rep The repository to bind againt, null if there is no repository available.
+ * @throws KettleXMLException
+ */
+ @Deprecated
+ public JobMeta(VariableSpace parentSpace, String fname, Repository rep, OverwritePrompter prompter)
+ throws KettleXMLException {
+ this(parentSpace, fname, rep, null, prompter);
+ }
+
+ /**
+ * Load the job from the XML file specified.
+ *
+ * @param fname The filename to load as a job
+ * @param rep The repository to bind againt, null if there is no repository available.
+ * @throws KettleXMLException
+ */
+ public JobMeta(VariableSpace parentSpace, String fname, Repository rep, IMetaStore metaStore,
+ OverwritePrompter prompter) throws KettleXMLException {
+ this.initializeVariablesFrom(parentSpace);
+ this.metaStore = metaStore;
+ try {
+ // OK, try to load using the VFS stuff...
+ Document doc = XMLHandler.loadXMLFile(KettleVFS.getFileObject(fname, this));
+ if (doc != null) {
+ // The jobnode
+ Node jobnode = XMLHandler.getSubNode(doc, XML_TAG);
+
+ loadXML(jobnode, fname, rep, metaStore, false, prompter);
+ } else {
+ throw new KettleXMLException(
+ BaseMessages.getString(PKG, "JobMeta.Exception.ErrorReadingFromXMLFile") + fname);
+ }
+ } catch (Exception e) {
+ throw new KettleXMLException(
+ BaseMessages.getString(PKG, "JobMeta.Exception.UnableToLoadJobFromXMLFile") + fname + "]", e);
+ }
+ }
+
+ /**
+ * Instantiates a new job meta.
+ *
+ * @param inputStream the input stream
+ * @param rep the rep
+ * @param prompter the prompter
+ * @throws KettleXMLException the kettle xml exception
+ */
+ public JobMeta(InputStream inputStream, Repository rep, OverwritePrompter prompter) throws KettleXMLException {
+ this();
+ Document doc = XMLHandler.loadXMLFile(inputStream, null, false, false);
+ loadXML(XMLHandler.getSubNode(doc, JobMeta.XML_TAG), rep, prompter);
+ }
+
+ /**
+ * Create a new JobMeta object by loading it from a a DOM node.
+ *
+ * @param jobnode The node to load from
+ * @param rep The reference to a repository to load additional information from
+ * @param prompter The prompter to use in case a shared object gets overwritten
+ * @throws KettleXMLException
+ */
+ public JobMeta(Node jobnode, Repository rep, OverwritePrompter prompter) throws KettleXMLException {
+ this();
+ loadXML(jobnode, rep, false, prompter);
+ }
+
+ /**
+ * Create a new JobMeta object by loading it from a a DOM node.
+ *
+ * @param jobnode The node to load from
+ * @param rep The reference to a repository to load additional information from
+ * @param ignoreRepositorySharedObjects Do not load shared objects, handled separately
+ * @param prompter The prompter to use in case a shared object gets overwritten
+ * @throws KettleXMLException
+ */
+ public JobMeta(Node jobnode, Repository rep, boolean ignoreRepositorySharedObjects, OverwritePrompter prompter)
+ throws KettleXMLException {
+ this();
+ loadXML(jobnode, rep, ignoreRepositorySharedObjects, prompter);
+ }
+
+ /**
+ * Checks if is rep reference.
+ *
+ * @return true, if is rep reference
+ */
+ public boolean isRepReference() {
+ return isRepReference(getFilename(), this.getName());
+ }
+
+ /**
+ * Checks if is file reference.
+ *
+ * @return true, if is file reference
+ */
+ public boolean isFileReference() {
+ return !isRepReference(getFilename(), this.getName());
+ }
+
+ /**
+ * Checks if is rep reference.
+ *
+ * @param fileName the file name
+ * @param transName the trans name
+ * @return true, if is rep reference
+ */
+ public static boolean isRepReference(String fileName, String transName) {
+ return Const.isEmpty(fileName) && !Const.isEmpty(transName);
+ }
+
+ /**
+ * Checks if is file reference.
+ *
+ * @param fileName the file name
+ * @param transName the trans name
+ * @return true, if is file reference
+ */
+ public static boolean isFileReference(String fileName, String transName) {
+ return !isRepReference(fileName, transName);
+ }
+
+ /**
+ * Load xml.
+ *
+ * @param jobnode the jobnode
+ * @param rep the rep
+ * @param prompter the prompter
+ * @throws KettleXMLException the kettle xml exception
+ */
+ public void loadXML(Node jobnode, Repository rep, OverwritePrompter prompter) throws KettleXMLException {
+ loadXML(jobnode, rep, false, prompter);
+ }
+
+ /**
+ * Load xml.
+ *
+ * @param jobnode the jobnode
+ * @param fname The filename
+ * @param rep the rep
+ * @param prompter the prompter
+ * @throws KettleXMLException the kettle xml exception
+ */
+ public void loadXML(Node jobnode, String fname, Repository rep, OverwritePrompter prompter)
+ throws KettleXMLException {
+ loadXML(jobnode, fname, rep, false, prompter);
+ }
+
+ /**
+ * Load a block of XML from an DOM node.
+ *
+ * @param jobnode The node to load from
+ * @param rep The reference to a repository to load additional information from
+ * @param ignoreRepositorySharedObjects Do not load shared objects, handled separately
+ * @param prompter The prompter to use in case a shared object gets overwritten
+ * @throws KettleXMLException
+ */
+ public void loadXML(Node jobnode, Repository rep, boolean ignoreRepositorySharedObjects, OverwritePrompter prompter)
+ throws KettleXMLException {
+ loadXML(jobnode, null, rep, ignoreRepositorySharedObjects, prompter);
+ }
+
+ /**
+ * Load a block of XML from an DOM node.
+ *
+ * @param jobnode The node to load from
+ * @param fname The filename
+ * @param rep The reference to a repository to load additional information from
+ * @param ignoreRepositorySharedObjects Do not load shared objects, handled separately
+ * @param prompter The prompter to use in case a shared object gets overwritten
+ * @throws KettleXMLException
+ * @deprecated
+ */
+ @Deprecated
+ public void loadXML(Node jobnode, String fname, Repository rep, boolean ignoreRepositorySharedObjects,
+ OverwritePrompter prompter) throws KettleXMLException {
+ loadXML(jobnode, fname, rep, null, ignoreRepositorySharedObjects, prompter);
+ }
+
+ /**
+ * Load a block of XML from an DOM node.
+ *
+ * @param jobnode The node to load from
+ * @param fname The filename
+ * @param rep The reference to a repository to load additional information from
+ * @param metaStore the MetaStore to use
+ * @param ignoreRepositorySharedObjects Do not load shared objects, handled separately
+ * @param prompter The prompter to use in case a shared object gets overwritten
+ * @throws KettleXMLException
+ */
+ public void loadXML(Node jobnode, String fname, Repository rep, IMetaStore metaStore,
+ boolean ignoreRepositorySharedObjects, OverwritePrompter prompter) throws KettleXMLException {
+ Props props = null;
+ if (Props.isInitialized()) {
+ props = Props.getInstance();
+ }
+
+ try {
+ // clear the jobs;
+ clear();
+
+ // If we are not using a repository, we are getting the job from a file
+ // Set the filename here so it can be used in variables for ALL aspects of the job FIX: PDI-8890
+ if (null == rep) {
+ setFilename(fname);
+ }
+
+ //
+ // get job info:
+ //
+ setName(XMLHandler.getTagValue(jobnode, "name"));
+
+ // Optionally load the repository directory...
+ //
+ if (rep != null) {
+ String directoryPath = XMLHandler.getTagValue(jobnode, "directory");
+ if (directoryPath != null) {
+ directory = rep.findDirectory(directoryPath);
+ if (directory == null) { // not found
+ directory = new RepositoryDirectory(); // The root as default
+ }
+ }
+ }
+
+ // description
+ description = XMLHandler.getTagValue(jobnode, "description");
+
+ // extended description
+ extendedDescription = XMLHandler.getTagValue(jobnode, "extended_description");
+
+ // job version
+ jobVersion = XMLHandler.getTagValue(jobnode, "job_version");
+
+ // job status
+ jobStatus = Const.toInt(XMLHandler.getTagValue(jobnode, "job_status"), -1);
+
+ // Created user/date
+ createdUser = XMLHandler.getTagValue(jobnode, "created_user");
+ String createDate = XMLHandler.getTagValue(jobnode, "created_date");
+
+ if (createDate != null) {
+ createdDate = XMLHandler.stringToDate(createDate);
+ }
+
+ // Changed user/date
+ modifiedUser = XMLHandler.getTagValue(jobnode, "modified_user");
+ String modDate = XMLHandler.getTagValue(jobnode, "modified_date");
+ if (modDate != null) {
+ modifiedDate = XMLHandler.stringToDate(modDate);
+ }
+
+ // Load the default list of databases
+ // Read objects from the shared XML file & the repository
+ try {
+ sharedObjectsFile = XMLHandler.getTagValue(jobnode, "shared_objects_file");
+ if (rep == null || ignoreRepositorySharedObjects) {
+ sharedObjects = readSharedObjects();
+ } else {
+ sharedObjects = rep.readJobMetaSharedObjects(this);
+ }
+ } catch (Exception e) {
+ LogChannel.GENERAL
+ .logError(BaseMessages.getString(PKG, "JobMeta.ErrorReadingSharedObjects.Message", e.toString()));
+ LogChannel.GENERAL.logError(Const.getStackTracker(e));
+ }
+
+ // Load the database connections, slave servers, cluster schemas & partition schemas into this object.
+ //
+ importFromMetaStore();
+
+ // Read the named parameters.
+ Node paramsNode = XMLHandler.getSubNode(jobnode, XML_TAG_PARAMETERS);
+ int nrParams = XMLHandler.countNodes(paramsNode, "parameter");
+
+ for (int i = 0; i < nrParams; i++) {
+ Node paramNode = XMLHandler.getSubNodeByNr(paramsNode, "parameter", i);
+
+ String paramName = XMLHandler.getTagValue(paramNode, "name");
+ String defValue = XMLHandler.getTagValue(paramNode, "default_value");
+ String descr = XMLHandler.getTagValue(paramNode, "description");
+
+ addParameterDefinition(paramName, defValue, descr);
+ }
+
+ //
+ // Read the database connections
+ //
+ int nr = XMLHandler.countNodes(jobnode, "connection");
+ Set privateDatabases = new HashSet(nr);
+ for (int i = 0; i < nr; i++) {
+ Node dbnode = XMLHandler.getSubNodeByNr(jobnode, "connection", i);
+ DatabaseMeta dbcon = new DatabaseMeta(dbnode);
+ dbcon.shareVariablesWith(this);
+ if (!dbcon.isShared()) {
+ privateDatabases.add(dbcon.getName());
+ }
+
+ DatabaseMeta exist = findDatabase(dbcon.getName());
+ if (exist == null) {
+ addDatabase(dbcon);
+ } else {
+ if (!exist.isShared()) {
+ // skip shared connections
+ if (shouldOverwrite(prompter, props,
+ BaseMessages.getString(PKG, "JobMeta.Dialog.ConnectionExistsOverWrite.Message", dbcon.getName()),
+ BaseMessages.getString(PKG, "JobMeta.Dialog.ConnectionExistsOverWrite.DontShowAnyMoreMessage"))) {
+ int idx = indexOfDatabase(exist);
+ removeDatabase(idx);
+ addDatabase(idx, dbcon);
+ }
+ }
+ }
+ }
+ setPrivateDatabases(privateDatabases);
+
+ // Read the slave servers...
+ //
+ Node slaveServersNode = XMLHandler.getSubNode(jobnode, XML_TAG_SLAVESERVERS);
+ int nrSlaveServers = XMLHandler.countNodes(slaveServersNode, SlaveServer.XML_TAG);
+ for (int i = 0; i < nrSlaveServers; i++) {
+ Node slaveServerNode = XMLHandler.getSubNodeByNr(slaveServersNode, SlaveServer.XML_TAG, i);
+ SlaveServer slaveServer = new SlaveServer(slaveServerNode);
+ slaveServer.shareVariablesWith(this);
+
+ // Check if the object exists and if it's a shared object.
+ // If so, then we will keep the shared version, not this one.
+ // The stored XML is only for backup purposes.
+ SlaveServer check = findSlaveServer(slaveServer.getName());
+ if (check != null) {
+ if (!check.isShared()) {
+ // we don't overwrite shared objects.
+ if (shouldOverwrite(prompter, props, BaseMessages
+ .getString(PKG, "JobMeta.Dialog.SlaveServerExistsOverWrite.Message", slaveServer.getName()),
+ BaseMessages.getString(PKG, "JobMeta.Dialog.ConnectionExistsOverWrite.DontShowAnyMoreMessage"))) {
+ addOrReplaceSlaveServer(slaveServer);
+ }
+ }
+ } else {
+ slaveServers.add(slaveServer);
+ }
+ }
+
+ /*
+ * Get the log database connection & log table
+ */
+ // Backward compatibility...
+ //
+ Node jobLogNode = XMLHandler.getSubNode(jobnode, JobLogTable.XML_TAG);
+ if (jobLogNode == null) {
+ // Load the XML
+ //
+ jobLogTable.setConnectionName(XMLHandler.getTagValue(jobnode, "logconnection"));
+ jobLogTable.setTableName(XMLHandler.getTagValue(jobnode, "logtable"));
+ jobLogTable.setBatchIdUsed("Y".equalsIgnoreCase(XMLHandler.getTagValue(jobnode, "use_batchid")));
+ jobLogTable.setLogFieldUsed("Y".equalsIgnoreCase(XMLHandler.getTagValue(jobnode, "use_logfield")));
+ jobLogTable.findField(JobLogTable.ID.CHANNEL_ID).setEnabled(false);
+ jobLogTable.findField(JobLogTable.ID.LINES_REJECTED).setEnabled(false);
+ } else {
+ jobLogTable.loadXML(jobLogNode, databases, null);
+ }
+
+ Node channelLogTableNode = XMLHandler.getSubNode(jobnode, ChannelLogTable.XML_TAG);
+ if (channelLogTableNode != null) {
+ channelLogTable.loadXML(channelLogTableNode, databases, null);
+ }
+ jobEntryLogTable.loadXML(jobnode, databases, null);
+
+ for (LogTableInterface extraLogTable : extraLogTables) {
+ extraLogTable.loadXML(jobnode, databases, null);
+ }
+
+ batchIdPassed = "Y".equalsIgnoreCase(XMLHandler.getTagValue(jobnode, "pass_batchid"));
+
+ /*
+ * read the job entries...
+ */
+ Node entriesnode = XMLHandler.getSubNode(jobnode, "entries");
+ int tr = XMLHandler.countNodes(entriesnode, "entry");
+ for (int i = 0; i < tr; i++) {
+ Node entrynode = XMLHandler.getSubNodeByNr(entriesnode, "entry", i);
+ // System.out.println("Reading entry:\n"+entrynode);
+
+ JobEntryCopy je = new JobEntryCopy(entrynode, databases, slaveServers, rep, metaStore);
+
+ if (je.isSpecial() && je.isMissing()) {
+ addMissingEntry((MissingEntry) je.getEntry());
+ }
+ JobEntryCopy prev = findJobEntry(je.getName(), 0, true);
+ if (prev != null) {
+ // See if the #0 (root entry) already exists!
+ //
+ if (je.getNr() == 0) {
+
+ // Replace previous version with this one: remove it first
+ //
+ int idx = indexOfJobEntry(prev);
+ removeJobEntry(idx);
+
+ } else if (je.getNr() > 0) {
+
+ // Use previously defined JobEntry info!
+ //
+ je.setEntry(prev.getEntry());
+
+ // See if entry already exists...
+ prev = findJobEntry(je.getName(), je.getNr(), true);
+ if (prev != null) {
+ // remove the old one!
+ //
+ int idx = indexOfJobEntry(prev);
+ removeJobEntry(idx);
+ }
+ }
+ }
+ // Add the JobEntryCopy...
+ addJobEntry(je);
+ }
+
+ Node hopsnode = XMLHandler.getSubNode(jobnode, "hops");
+ int ho = XMLHandler.countNodes(hopsnode, "hop");
+ for (int i = 0; i < ho; i++) {
+ Node hopnode = XMLHandler.getSubNodeByNr(hopsnode, "hop", i);
+ JobHopMeta hi = new JobHopMeta(hopnode, this);
+ jobhops.add(hi);
+ }
+
+ // Read the notes...
+ Node notepadsnode = XMLHandler.getSubNode(jobnode, "notepads");
+ int nrnotes = XMLHandler.countNodes(notepadsnode, "notepad");
+ for (int i = 0; i < nrnotes; i++) {
+ Node notepadnode = XMLHandler.getSubNodeByNr(notepadsnode, "notepad", i);
+ NotePadMeta ni = new NotePadMeta(notepadnode);
+ notes.add(ni);
+ }
+
+ // Load the attribute groups map
+ //
+ attributesMap = AttributesUtil.loadAttributes(XMLHandler.getSubNode(jobnode, AttributesUtil.XML_TAG));
+
+ ExtensionPointHandler.callExtensionPoint(LogChannel.GENERAL, KettleExtensionPoint.JobMetaLoaded.id, this);
+
+ clearChanged();
+ } catch (Exception e) {
+ throw new KettleXMLException(BaseMessages.getString(PKG, "JobMeta.Exception.UnableToLoadJobFromXMLNode"), e);
+ } finally {
+ setInternalKettleVariables();
+ }
+ }
+
+ /**
+ * Read shared objects.
+ *
+ * @return the shared objects
+ * @throws KettleException the kettle exception
+ */
+ public SharedObjects readSharedObjects() throws KettleException {
+ // Extract the shared steps, connections, etc. using the SharedObjects
+ // class
+ //
+ String soFile = environmentSubstitute(sharedObjectsFile);
+ SharedObjects sharedObjects = new SharedObjects(soFile);
+ Map objectsMap = sharedObjects.getObjectsMap();
+
+ // First read the databases...
+ // We read databases & slaves first because there might be dependencies
+ // that need to be resolved.
+ //
+ for (SharedObjectInterface object : objectsMap.values()) {
+ if (object instanceof DatabaseMeta) {
+ DatabaseMeta databaseMeta = (DatabaseMeta) object;
+ databaseMeta.shareVariablesWith(this);
+ addOrReplaceDatabase(databaseMeta);
+ } else if (object instanceof SlaveServer) {
+ SlaveServer slaveServer = (SlaveServer) object;
+ slaveServer.shareVariablesWith(this);
+ addOrReplaceSlaveServer(slaveServer);
+ }
+ }
+
+ return sharedObjects;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.EngineMetaInterface#saveSharedObjects()
+ */
+ public void saveSharedObjects() throws KettleException {
+ try {
+ // First load all the shared objects...
+ String soFile = environmentSubstitute(sharedObjectsFile);
+ SharedObjects sharedObjects = new SharedObjects(soFile);
+
+ // Now overwrite the objects in there
+ List shared = new ArrayList();
+ shared.addAll(databases);
+ shared.addAll(slaveServers);
+
+ // The databases connections...
+ for (int i = 0; i < shared.size(); i++) {
+ SharedObjectInterface sharedObject = (SharedObjectInterface) shared.get(i);
+ if (sharedObject.isShared()) {
+ sharedObjects.storeObject(sharedObject);
+ }
+ }
+
+ // Save the objects
+ sharedObjects.saveToFile();
+ } catch (Exception e) {
+ throw new KettleException("Unable to save shared ojects", e);
+ }
+ }
+
+ /**
+ * Gets the job entry copy.
+ *
+ * @param x the x
+ * @param y the y
+ * @param iconsize the iconsize
+ * @return the job entry copy
+ */
+ public JobEntryCopy getJobEntryCopy(int x, int y, int iconsize) {
+ int i, s;
+ s = nrJobEntries();
+ for (i = s - 1; i >= 0; i--) {
+ // Back to front because drawing goes from start to end
+
+ JobEntryCopy je = getJobEntry(i);
+ Point p = je.getLocation();
+ if (p != null) {
+ if (x >= p.x && x <= p.x + iconsize && y >= p.y && y <= p.y + iconsize) {
+ return je;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Nr job entries.
+ *
+ * @return the int
+ */
+ public int nrJobEntries() {
+ return jobcopies.size();
+ }
+
+ /**
+ * Nr job hops.
+ *
+ * @return the int
+ */
+ public int nrJobHops() {
+ return jobhops.size();
+ }
+
+ /**
+ * Gets the job hop.
+ *
+ * @param i the i
+ * @return the job hop
+ */
+ public JobHopMeta getJobHop(int i) {
+ return jobhops.get(i);
+ }
+
+ /**
+ * Gets the job entry.
+ *
+ * @param i the i
+ * @return the job entry
+ */
+ public JobEntryCopy getJobEntry(int i) {
+ return jobcopies.get(i);
+ }
+
+ /**
+ * Adds the job entry.
+ *
+ * @param je the je
+ */
+ public void addJobEntry(JobEntryCopy je) {
+ jobcopies.add(je);
+ je.setParentJobMeta(this);
+ setChanged();
+ }
+
+ /**
+ * Adds the job hop.
+ *
+ * @param hi the hi
+ */
+ public void addJobHop(JobHopMeta hi) {
+ jobhops.add(hi);
+ setChanged();
+ }
+
+ /**
+ * Adds the job entry.
+ *
+ * @param p the p
+ * @param si the si
+ */
+ public void addJobEntry(int p, JobEntryCopy si) {
+ jobcopies.add(p, si);
+ changedEntries = true;
+ }
+
+ /**
+ * Adds the job hop.
+ *
+ * @param p the p
+ * @param hi the hi
+ */
+ public void addJobHop(int p, JobHopMeta hi) {
+ try {
+ jobhops.add(p, hi);
+ } catch (IndexOutOfBoundsException e) {
+ jobhops.add(hi);
+ }
+ changedHops = true;
+ }
+
+ /**
+ * Removes the job entry.
+ *
+ * @param i the i
+ */
+ public void removeJobEntry(int i) {
+ JobEntryCopy deleted = jobcopies.remove(i);
+ if (deleted != null) {
+ if (deleted.getEntry() instanceof MissingEntry) {
+ removeMissingEntry((MissingEntry) deleted.getEntry());
+ }
+ }
+ setChanged();
+ }
+
+ /**
+ * Removes the job hop.
+ *
+ * @param i the i
+ */
+ public void removeJobHop(int i) {
+ jobhops.remove(i);
+ setChanged();
+ }
+
+ /**
+ * Removes a hop from the transformation. Also marks that the
+ * transformation's hops have changed.
+ *
+ * @param hop The hop to remove from the list of hops
+ */
+ public void removeJobHop(JobHopMeta hop) {
+ jobhops.remove(hop);
+ setChanged();
+ }
+
+ /**
+ * Index of job hop.
+ *
+ * @param he the he
+ * @return the int
+ */
+ public int indexOfJobHop(JobHopMeta he) {
+ return jobhops.indexOf(he);
+ }
+
+ /**
+ * Index of job entry.
+ *
+ * @param ge the ge
+ * @return the int
+ */
+ public int indexOfJobEntry(JobEntryCopy ge) {
+ return jobcopies.indexOf(ge);
+ }
+
+ /**
+ * Sets the job entry.
+ *
+ * @param idx the idx
+ * @param jec the jec
+ */
+ public void setJobEntry(int idx, JobEntryCopy jec) {
+ jobcopies.set(idx, jec);
+ }
+
+ /**
+ * Find an existing JobEntryCopy by it's name and number
+ *
+ * @param name The name of the job entry copy
+ * @param nr The number of the job entry copy
+ * @return The JobEntryCopy or null if nothing was found!
+ */
+ public JobEntryCopy findJobEntry(String name, int nr, boolean searchHiddenToo) {
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy jec = getJobEntry(i);
+ if (jec.getName().equalsIgnoreCase(name) && jec.getNr() == nr) {
+ if (searchHiddenToo || jec.isDrawn()) {
+ return jec;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find job entry.
+ *
+ * @param full_name_nr the full_name_nr
+ * @return the job entry copy
+ */
+ public JobEntryCopy findJobEntry(String full_name_nr) {
+ int i;
+ for (i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy jec = getJobEntry(i);
+ JobEntryInterface je = jec.getEntry();
+ if (je.toString().equalsIgnoreCase(full_name_nr)) {
+ return jec;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find job hop.
+ *
+ * @param name the name
+ * @return the job hop meta
+ */
+ public JobHopMeta findJobHop(String name) {
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.toString().equalsIgnoreCase(name)) {
+ return hi;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find job hop from.
+ *
+ * @param jge the jge
+ * @return the job hop meta
+ */
+ public JobHopMeta findJobHopFrom(JobEntryCopy jge) {
+ if (jge != null) {
+ for (JobHopMeta hi : jobhops) {
+
+ // Return the first we find!
+ //
+ if (hi != null && (hi.getFromEntry() != null) && hi.getFromEntry().equals(jge)) {
+ return hi;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find job hop.
+ *
+ * @param from the from
+ * @param to the to
+ * @return the job hop meta
+ */
+ public JobHopMeta findJobHop(JobEntryCopy from, JobEntryCopy to) {
+ return findJobHop(from, to, false);
+ }
+
+ /**
+ * Find job hop.
+ *
+ * @param from the from
+ * @param to the to
+ * @param includeDisabled the include disabled
+ * @return the job hop meta
+ */
+ public JobHopMeta findJobHop(JobEntryCopy from, JobEntryCopy to, boolean includeDisabled) {
+ for (JobHopMeta hi : jobhops) {
+ if (hi.isEnabled() || includeDisabled) {
+ if (hi != null && hi.getFromEntry() != null && hi.getToEntry() != null && hi.getFromEntry().equals(from)
+ && hi.getToEntry().equals(to)) {
+ return hi;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find job hop to.
+ *
+ * @param jge the jge
+ * @return the job hop meta
+ */
+ public JobHopMeta findJobHopTo(JobEntryCopy jge) {
+ for (JobHopMeta hi : jobhops) {
+ if (hi != null && hi.getToEntry() != null && hi.getToEntry().equals(jge)) {
+ // Return the first!
+ return hi;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find nr prev job entries.
+ *
+ * @param from the from
+ * @return the int
+ */
+ public int findNrPrevJobEntries(JobEntryCopy from) {
+ return findNrPrevJobEntries(from, false);
+ }
+
+ /**
+ * Find prev job entry.
+ *
+ * @param to the to
+ * @param nr the nr
+ * @return the job entry copy
+ */
+ public JobEntryCopy findPrevJobEntry(JobEntryCopy to, int nr) {
+ return findPrevJobEntry(to, nr, false);
+ }
+
+ /**
+ * Find nr prev job entries.
+ *
+ * @param to the to
+ * @param info the info
+ * @return the int
+ */
+ public int findNrPrevJobEntries(JobEntryCopy to, boolean info) {
+ int count = 0;
+
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.isEnabled() && hi.getToEntry().equals(to)) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Find prev job entry.
+ *
+ * @param to the to
+ * @param nr the nr
+ * @param info the info
+ * @return the job entry copy
+ */
+ public JobEntryCopy findPrevJobEntry(JobEntryCopy to, int nr, boolean info) {
+ int count = 0;
+
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.isEnabled() && hi.getToEntry().equals(to)) {
+ if (count == nr) {
+ return hi.getFromEntry();
+ }
+ count++;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find nr next job entries.
+ *
+ * @param from the from
+ * @return the int
+ */
+ public int findNrNextJobEntries(JobEntryCopy from) {
+ int count = 0;
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.isEnabled() && (hi.getFromEntry() != null) && hi.getFromEntry().equals(from)) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Find next job entry.
+ *
+ * @param from the from
+ * @param cnt the cnt
+ * @return the job entry copy
+ */
+ public JobEntryCopy findNextJobEntry(JobEntryCopy from, int cnt) {
+ int count = 0;
+
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.isEnabled() && (hi.getFromEntry() != null) && hi.getFromEntry().equals(from)) {
+ if (count == cnt) {
+ return hi.getToEntry();
+ }
+ count++;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Checks for loop.
+ *
+ * @param entry the entry
+ * @return true, if successful
+ */
+ public boolean hasLoop(JobEntryCopy entry) {
+ clearLoopCache();
+ return hasLoop(entry, null, true) || hasLoop(entry, null, false);
+ }
+
+ /**
+ * Checks for loop.
+ *
+ * @param entry the entry
+ * @param lookup the lookup
+ * @return true, if successful
+ */
+ public boolean hasLoop(JobEntryCopy entry, JobEntryCopy lookup, boolean info) {
+ String cacheKey =
+ entry.getName() + " - " + (lookup != null ? lookup.getName() : "") + " - " + (info ? "true" : "false");
+
+ Boolean loop = loopCache.get(cacheKey);
+ if (loop != null) {
+ return loop.booleanValue();
+ }
+
+ boolean hasLoop = false;
+
+ int nr = findNrPrevJobEntries(entry, info);
+ for (int i = 0; i < nr && !hasLoop; i++) {
+ JobEntryCopy prevJobMeta = findPrevJobEntry(entry, i, info);
+ if (prevJobMeta != null) {
+ if (prevJobMeta.equals(entry)) {
+ hasLoop = true;
+ break; // no need to check more but caching this one below
+ } else if (prevJobMeta.equals(lookup)) {
+ hasLoop = true;
+ break; // no need to check more but caching this one below
+ } else if (hasLoop(prevJobMeta, lookup == null ? entry : lookup, info)) {
+ hasLoop = true;
+ break; // no need to check more but caching this one below
+ }
+ }
+ }
+ // Store in the cache...
+ //
+ loopCache.put(cacheKey, Boolean.valueOf(hasLoop));
+ return hasLoop;
+ }
+
+ /**
+ * Clears the loop cache.
+ */
+ private void clearLoopCache() {
+ loopCache.clear();
+ }
+
+ /**
+ * Checks if is entry used in hops.
+ *
+ * @param jge the jge
+ * @return true, if is entry used in hops
+ */
+ public boolean isEntryUsedInHops(JobEntryCopy jge) {
+ JobHopMeta fr = findJobHopFrom(jge);
+ JobHopMeta to = findJobHopTo(jge);
+ if (fr != null || to != null) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Count entries.
+ *
+ * @param name the name
+ * @return the int
+ */
+ public int countEntries(String name) {
+ int count = 0;
+ int i;
+ for (i = 0; i < nrJobEntries(); i++) {
+ // Look at all the hops;
+
+ JobEntryCopy je = getJobEntry(i);
+ if (je.getName().equalsIgnoreCase(name)) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Find unused nr.
+ *
+ * @param name the name
+ * @return the int
+ */
+ public int findUnusedNr(String name) {
+ int nr = 1;
+ JobEntryCopy je = findJobEntry(name, nr, true);
+ while (je != null) {
+ nr++;
+ // log.logDebug("findUnusedNr()", "Trying unused nr: "+nr);
+ je = findJobEntry(name, nr, true);
+ }
+ return nr;
+ }
+
+ /**
+ * Find max nr.
+ *
+ * @param name the name
+ * @return the int
+ */
+ public int findMaxNr(String name) {
+ int max = 0;
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy je = getJobEntry(i);
+ if (je.getName().equalsIgnoreCase(name)) {
+ if (je.getNr() > max) {
+ max = je.getNr();
+ }
+ }
+ }
+ return max;
+ }
+
+ /**
+ * Proposes an alternative job entry name when the original already exists...
+ *
+ * @param entryname The job entry name to find an alternative for..
+ * @return The alternative stepname.
+ */
+ public String getAlternativeJobentryName(String entryname) {
+ String newname = entryname;
+ JobEntryCopy jec = findJobEntry(newname);
+ int nr = 1;
+ while (jec != null) {
+ nr++;
+ newname = entryname + " " + nr;
+ jec = findJobEntry(newname);
+ }
+
+ return newname;
+ }
+
+ /**
+ * Gets the all job graph entries.
+ *
+ * @param name the name
+ * @return the all job graph entries
+ */
+ public JobEntryCopy[] getAllJobGraphEntries(String name) {
+ int count = 0;
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy je = getJobEntry(i);
+ if (je.getName().equalsIgnoreCase(name)) {
+ count++;
+ }
+ }
+ JobEntryCopy[] retval = new JobEntryCopy[count];
+
+ count = 0;
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy je = getJobEntry(i);
+ if (je.getName().equalsIgnoreCase(name)) {
+ retval[count] = je;
+ count++;
+ }
+ }
+ return retval;
+ }
+
+ /**
+ * Gets the all job hops using.
+ *
+ * @param name the name
+ * @return the all job hops using
+ */
+ public JobHopMeta[] getAllJobHopsUsing(String name) {
+ List hops = new ArrayList();
+
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.getFromEntry() != null && hi.getToEntry() != null) {
+ if (hi.getFromEntry().getName().equalsIgnoreCase(name) || hi.getToEntry().getName()
+ .equalsIgnoreCase(name)) {
+ hops.add(hi);
+ }
+ }
+ }
+ return hops.toArray(new JobHopMeta[hops.size()]);
+ }
+
+ public boolean isPathExist(JobEntryInterface from, JobEntryInterface to) {
+ for (JobHopMeta hi : jobhops) {
+ if (hi.getFromEntry() != null && hi.getToEntry() != null) {
+ if (hi.getFromEntry().getName().equalsIgnoreCase(from.getName())) {
+ if (hi.getToEntry().getName().equalsIgnoreCase(to.getName())) {
+ return true;
+ }
+ if (isPathExist(hi.getToEntry().getEntry(), to)) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Select all.
+ */
+ public void selectAll() {
+ int i;
+ for (i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy ce = getJobEntry(i);
+ ce.setSelected(true);
+ }
+ for (i = 0; i < nrNotes(); i++) {
+ NotePadMeta ni = getNote(i);
+ ni.setSelected(true);
+ }
+ setChanged();
+ notifyObservers("refreshGraph");
+ }
+
+ /**
+ * Unselect all.
+ */
+ public void unselectAll() {
+ int i;
+ for (i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy ce = getJobEntry(i);
+ ce.setSelected(false);
+ }
+ for (i = 0; i < nrNotes(); i++) {
+ NotePadMeta ni = getNote(i);
+ ni.setSelected(false);
+ }
+ }
+
+ /**
+ * Gets the maximum.
+ *
+ * @return the maximum
+ */
+ public Point getMaximum() {
+ int maxx = 0, maxy = 0;
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy entry = getJobEntry(i);
+ Point loc = entry.getLocation();
+ if (loc.x > maxx) {
+ maxx = loc.x;
+ }
+ if (loc.y > maxy) {
+ maxy = loc.y;
+ }
+ }
+ for (int i = 0; i < nrNotes(); i++) {
+ NotePadMeta ni = getNote(i);
+ Point loc = ni.getLocation();
+ if (loc.x + ni.width > maxx) {
+ maxx = loc.x + ni.width;
+ }
+ if (loc.y + ni.height > maxy) {
+ maxy = loc.y + ni.height;
+ }
+ }
+
+ return new Point(maxx + 100, maxy + 100);
+ }
+
+ /**
+ * Get the minimum point on the canvas of a job
+ *
+ * @return Minimum coordinate of a step in the job
+ */
+ public Point getMinimum() {
+ int minx = Integer.MAX_VALUE;
+ int miny = Integer.MAX_VALUE;
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy jobEntryCopy = getJobEntry(i);
+ Point loc = jobEntryCopy.getLocation();
+ if (loc.x < minx) {
+ minx = loc.x;
+ }
+ if (loc.y < miny) {
+ miny = loc.y;
+ }
+ }
+ for (int i = 0; i < nrNotes(); i++) {
+ NotePadMeta notePadMeta = getNote(i);
+ Point loc = notePadMeta.getLocation();
+ if (loc.x < minx) {
+ minx = loc.x;
+ }
+ if (loc.y < miny) {
+ miny = loc.y;
+ }
+ }
+
+ if (minx > 20) {
+ minx -= 20;
+ } else {
+ minx = 0;
+ }
+ if (miny > 20) {
+ miny -= 20;
+ } else {
+ miny = 0;
+ }
+
+ return new Point(minx, miny);
+ }
+
+ /**
+ * Gets the selected locations.
+ *
+ * @return the selected locations
+ */
+ public Point[] getSelectedLocations() {
+ List selectedEntries = getSelectedEntries();
+ Point[] retval = new Point[selectedEntries.size()];
+ for (int i = 0; i < retval.length; i++) {
+ JobEntryCopy si = selectedEntries.get(i);
+ Point p = si.getLocation();
+ retval[i] = new Point(p.x, p.y); // explicit copy of location
+ }
+ return retval;
+ }
+
+ /**
+ * Get all the selected note locations
+ *
+ * @return The selected step and notes locations.
+ */
+ public Point[] getSelectedNoteLocations() {
+ List points = new ArrayList();
+
+ for (NotePadMeta ni : getSelectedNotes()) {
+ Point p = ni.getLocation();
+ points.add(new Point(p.x, p.y)); // explicit copy of location
+ }
+
+ return points.toArray(new Point[points.size()]);
+ }
+
+ /**
+ * Gets the selected entries.
+ *
+ * @return the selected entries
+ */
+ public List getSelectedEntries() {
+ List selection = new ArrayList();
+ for (JobEntryCopy je : jobcopies) {
+ if (je.isSelected()) {
+ selection.add(je);
+ }
+ }
+ return selection;
+ }
+
+ /**
+ * Gets the entry indexes.
+ *
+ * @param entries the entries
+ * @return the entry indexes
+ */
+ public int[] getEntryIndexes(List entries) {
+ int[] retval = new int[entries.size()];
+
+ for (int i = 0; i < entries.size(); i++) {
+ retval[i] = indexOfJobEntry(entries.get(i));
+ }
+
+ return retval;
+ }
+
+ /**
+ * Find start.
+ *
+ * @return the job entry copy
+ */
+ public JobEntryCopy findStart() {
+ for (int i = 0; i < nrJobEntries(); i++) {
+ if (getJobEntry(i).isStart()) {
+ return getJobEntry(i);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets a textual representation of the job. If its name has been set, it will be returned, otherwise the classname is
+ * returned.
+ *
+ * @return the textual representation of the job.
+ */
+ public String toString() {
+ if (!Const.isEmpty(filename)) {
+ if (Const.isEmpty(name)) {
+ return filename;
+ } else {
+ return filename + " : " + name;
+ }
+ }
+
+ if (name != null) {
+ if (directory != null) {
+ String path = directory.getPath();
+ if (path.endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ return path + name;
+ } else {
+ return path + RepositoryDirectory.DIRECTORY_SEPARATOR + name;
+ }
+ } else {
+ return name;
+ }
+ } else {
+ return JobMeta.class.getName();
+ }
+ }
+
+ /**
+ * Gets the boolean value of batch id passed.
+ *
+ * @return Returns the batchIdPassed.
+ */
+ public boolean isBatchIdPassed() {
+ return batchIdPassed;
+ }
+
+ /**
+ * Sets the batch id passed.
+ *
+ * @param batchIdPassed The batchIdPassed to set.
+ */
+ public void setBatchIdPassed(boolean batchIdPassed) {
+ this.batchIdPassed = batchIdPassed;
+ }
+
+ public List getSQLStatements(Repository repository, ProgressMonitorListener monitor)
+ throws KettleException {
+ return getSQLStatements(repository, null, monitor);
+ }
+
+ /**
+ * Builds a list of all the SQL statements that this transformation needs in order to work properly.
+ *
+ * @return An ArrayList of SQLStatement objects.
+ */
+ public List getSQLStatements(Repository repository, IMetaStore metaStore,
+ ProgressMonitorListener monitor) throws KettleException {
+ if (monitor != null) {
+ monitor
+ .beginTask(BaseMessages.getString(PKG, "JobMeta.Monitor.GettingSQLNeededForThisJob"), nrJobEntries() + 1);
+ }
+ List stats = new ArrayList();
+
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy copy = getJobEntry(i);
+ if (monitor != null) {
+ monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.GettingSQLForJobEntryCopy") + copy + "]");
+ }
+ stats.addAll(copy.getEntry().getSQLStatements(repository, metaStore, this));
+ stats.addAll(compatibleGetEntrySQLStatements(copy.getEntry(), repository));
+ stats.addAll(compatibleGetEntrySQLStatements(copy.getEntry(), repository, this));
+ if (monitor != null) {
+ monitor.worked(1);
+ }
+ }
+
+ // Also check the sql for the logtable...
+ if (monitor != null) {
+ monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.GettingSQLStatementsForJobLogTables"));
+ }
+ if (jobLogTable.getDatabaseMeta() != null && !Const.isEmpty(jobLogTable.getTableName())) {
+ Database db = new Database(this, jobLogTable.getDatabaseMeta());
+ try {
+ db.connect();
+ RowMetaInterface fields = jobLogTable.getLogRecord(LogStatus.START, null, null).getRowMeta();
+ String sql = db.getDDL(jobLogTable.getTableName(), fields);
+ if (sql != null && sql.length() > 0) {
+ SQLStatement stat = new SQLStatement(BaseMessages.getString(PKG, "JobMeta.SQLFeedback.ThisJob"),
+ jobLogTable.getDatabaseMeta(), sql);
+ stats.add(stat);
+ }
+ } catch (KettleDatabaseException dbe) {
+ SQLStatement stat = new SQLStatement(BaseMessages.getString(PKG, "JobMeta.SQLFeedback.ThisJob"),
+ jobLogTable.getDatabaseMeta(), null);
+ stat.setError(
+ BaseMessages.getString(PKG, "JobMeta.SQLFeedback.ErrorObtainingJobLogTableInfo") + dbe.getMessage());
+ stats.add(stat);
+ } finally {
+ db.disconnect();
+ }
+ }
+ if (monitor != null) {
+ monitor.worked(1);
+ }
+ if (monitor != null) {
+ monitor.done();
+ }
+
+ return stats;
+ }
+
+ @SuppressWarnings("deprecation")
+ private Collection compatibleGetEntrySQLStatements(JobEntryInterface entry,
+ Repository repository, VariableSpace variableSpace) throws KettleException {
+ return entry.getSQLStatements(repository, variableSpace);
+ }
+
+ @SuppressWarnings("deprecation")
+ private Collection compatibleGetEntrySQLStatements(JobEntryInterface entry,
+ Repository repository) throws KettleException {
+ return entry.getSQLStatements(repository);
+ }
+
+ /**
+ * Gets the arguments used for this job.
+ *
+ * @return Returns the arguments.
+ * @deprecated Moved to the Job class
+ */
+ @Deprecated
+ public String[] getArguments() {
+ return arguments;
+ }
+
+ /**
+ * Sets the arguments.
+ *
+ * @param arguments The arguments to set.
+ * @deprecated moved to the job class
+ */
+ @Deprecated
+ public void setArguments(String[] arguments) {
+ this.arguments = arguments;
+ }
+
+ /**
+ * Get a list of all the strings used in this job.
+ *
+ * @return A list of StringSearchResult with strings used in the job
+ */
+ public List getStringList(boolean searchSteps, boolean searchDatabases, boolean searchNotes) {
+ List stringList = new ArrayList();
+
+ if (searchSteps) {
+ // Loop over all steps in the transformation and see what the used
+ // vars are...
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy entryMeta = getJobEntry(i);
+ stringList.add(new StringSearchResult(entryMeta.getName(), entryMeta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.JobEntryName")));
+ if (entryMeta.getDescription() != null) {
+ stringList.add(new StringSearchResult(entryMeta.getDescription(), entryMeta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.JobEntryDescription")));
+ }
+ JobEntryInterface metaInterface = entryMeta.getEntry();
+ StringSearcher.findMetaData(metaInterface, 1, stringList, entryMeta, this);
+ }
+ }
+
+ // Loop over all steps in the transformation and see what the used vars
+ // are...
+ if (searchDatabases) {
+ for (int i = 0; i < nrDatabases(); i++) {
+ DatabaseMeta meta = getDatabase(i);
+ stringList.add(new StringSearchResult(meta.getName(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseConnectionName")));
+ if (meta.getHostname() != null) {
+ stringList.add(new StringSearchResult(meta.getHostname(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseHostName")));
+ }
+ if (meta.getDatabaseName() != null) {
+ stringList.add(new StringSearchResult(meta.getDatabaseName(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseName")));
+ }
+ if (meta.getUsername() != null) {
+ stringList.add(new StringSearchResult(meta.getUsername(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseUsername")));
+ }
+ if (meta.getPluginId() != null) {
+ stringList.add(new StringSearchResult(meta.getPluginId(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseTypeDescription")));
+ }
+ if (meta.getDatabasePortNumberString() != null) {
+ stringList.add(new StringSearchResult(meta.getDatabasePortNumberString(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabasePort")));
+ }
+ if (meta.getServername() != null) {
+ stringList.add(new StringSearchResult(meta.getServername(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabaseServer")));
+ }
+ // if ( includePasswords )
+ // {
+ if (meta.getPassword() != null) {
+ stringList.add(new StringSearchResult(meta.getPassword(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.DatabasePassword")));
+ // }
+ }
+ }
+ }
+
+ // Loop over all steps in the transformation and see what the used vars
+ // are...
+ if (searchNotes) {
+ for (int i = 0; i < nrNotes(); i++) {
+ NotePadMeta meta = getNote(i);
+ if (meta.getNote() != null) {
+ stringList.add(new StringSearchResult(meta.getNote(), meta, this,
+ BaseMessages.getString(PKG, "JobMeta.SearchMetadata.NotepadText")));
+ }
+ }
+ }
+
+ return stringList;
+ }
+
+ /**
+ * Gets the used variables.
+ *
+ * @return the used variables
+ */
+ public List getUsedVariables() {
+ // Get the list of Strings.
+ List stringList = getStringList(true, true, false);
+
+ List varList = new ArrayList();
+
+ // Look around in the strings, see what we find...
+ for (StringSearchResult result : stringList) {
+ StringUtil.getUsedVariables(result.getString(), varList, false);
+ }
+
+ return varList;
+ }
+
+ /**
+ * Have job entries changed.
+ *
+ * @return true, if successful
+ */
+ public boolean haveJobEntriesChanged() {
+ if (changedEntries) {
+ return true;
+ }
+
+ for (int i = 0; i < nrJobEntries(); i++) {
+ JobEntryCopy entry = getJobEntry(i);
+ if (entry.hasChanged()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Have job hops changed.
+ *
+ * @return true, if successful
+ */
+ public boolean haveJobHopsChanged() {
+ if (changedHops) {
+ return true;
+ }
+
+ for (JobHopMeta hi : jobhops) {
+ // Look at all the hops
+
+ if (hi.hasChanged()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Gets the version of the job.
+ *
+ * @return The version of the job
+ */
+ public String getJobversion() {
+ return jobVersion;
+ }
+
+ /**
+ * Gets the status of the job.
+ *
+ * @return the status of the job
+ */
+ public int getJobstatus() {
+ return jobStatus;
+ }
+
+ /**
+ * Set the version of the job.
+ *
+ * @param jobVersion The new version description of the job
+ */
+ public void setJobversion(String jobVersion) {
+ this.jobVersion = jobVersion;
+ }
+
+ /**
+ * Set the status of the job.
+ *
+ * @param jobStatus The new status description of the job
+ */
+ public void setJobstatus(int jobStatus) {
+ this.jobStatus = jobStatus;
+ }
+
+ /**
+ * Find a jobentry with a certain ID in a list of job entries.
+ *
+ * @param jobentries The List of jobentries
+ * @param id_jobentry The id of the jobentry
+ * @return The JobEntry object if one was found, null otherwise.
+ */
+ public static final JobEntryInterface findJobEntry(List jobentries, ObjectId id_jobentry) {
+ if (jobentries == null) {
+ return null;
+ }
+
+ for (JobEntryInterface je : jobentries) {
+ if (je.getObjectId() != null && je.getObjectId().equals(id_jobentry)) {
+ return je;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find a jobentrycopy with a certain ID in a list of job entry copies.
+ *
+ * @param jobcopies The List of jobentry copies
+ * @param id_jobentry_copy The id of the jobentry copy
+ * @return The JobEntryCopy object if one was found, null otherwise.
+ */
+ public static final JobEntryCopy findJobEntryCopy(List jobcopies, ObjectId id_jobentry_copy) {
+ if (jobcopies == null) {
+ return null;
+ }
+
+ for (JobEntryCopy jec : jobcopies) {
+ if (jec.getObjectId() != null && jec.getObjectId().equals(id_jobentry_copy)) {
+ return jec;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method sets various internal kettle variables that can be used by the transformation.
+ */
+ @Override
+ public void setInternalKettleVariables(VariableSpace var) {
+ setInternalFilenameKettleVariables(var);
+ setInternalNameKettleVariable(var);
+
+ // The name of the directory in the repository
+ variables
+ .setVariable(Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY, directory != null ? directory.getPath() : "");
+
+ boolean hasRepoDir = getRepositoryDirectory() != null && getRepository() != null;
+
+ // setup fallbacks
+ if (hasRepoDir) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY,
+ variables.getVariable(Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY));
+ } else {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY,
+ variables.getVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY));
+ }
+
+ variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, variables.getVariable(
+ repository != null ? Const.INTERNAL_VARIABLE_JOB_REPOSITORY_DIRECTORY
+ : Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY));
+ }
+
+ /**
+ * Sets the internal name kettle variable.
+ *
+ * @param var the new internal name kettle variable
+ */
+ @Override
+ protected void setInternalNameKettleVariable(VariableSpace var) {
+ // The name of the job
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_NAME, Const.NVL(name, ""));
+ }
+
+ /**
+ * Sets the internal filename kettle variables.
+ *
+ * @param var the new internal filename kettle variables
+ */
+ @Override
+ protected void setInternalFilenameKettleVariables(VariableSpace var) {
+ if (filename != null) {
+ // we have a filename that's defined.
+ try {
+ FileObject fileObject = KettleVFS.getFileObject(filename, var);
+ FileName fileName = fileObject.getName();
+
+ // The filename of the job
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, fileName.getBaseName());
+
+ // The directory of the job
+ FileName fileDir = fileName.getParent();
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, fileDir.getURI());
+ } catch (Exception e) {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, "");
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, "");
+ }
+ } else {
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, "");
+ variables.setVariable(Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, "");
+ }
+ }
+
+ @Deprecated
+ public void checkJobEntries(List remarks, boolean only_selected,
+ ProgressMonitorListener monitor) {
+ checkJobEntries(remarks, only_selected, monitor, this, null, null);
+ }
+
+ /**
+ * Check all job entries within the job. Each Job Entry has the opportunity to check their own settings.
+ *
+ * @param remarks List of CheckResult remarks inserted into by each JobEntry
+ * @param only_selected true if you only want to check the selected jobs
+ * @param monitor Progress monitor (not presently in use)
+ */
+ public void checkJobEntries(List remarks, boolean only_selected,
+ ProgressMonitorListener monitor, VariableSpace space, Repository repository, IMetaStore metaStore) {
+ remarks.clear(); // Empty remarks
+ if (monitor != null) {
+ monitor.beginTask(BaseMessages.getString(PKG, "JobMeta.Monitor.VerifyingThisJobEntryTask.Title"),
+ jobcopies.size() + 2);
+ }
+ boolean stop_checking = false;
+ for (int i = 0; i < jobcopies.size() && !stop_checking; i++) {
+ JobEntryCopy copy = jobcopies.get(i); // get the job entry copy
+ if ((!only_selected) || (only_selected && copy.isSelected())) {
+ JobEntryInterface entry = copy.getEntry();
+ if (entry != null) {
+ if (monitor != null) {
+ monitor
+ .subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.VerifyingJobEntry.Title", entry.getName()));
+ }
+ entry.check(remarks, this, space, repository, metaStore);
+ compatibleEntryCheck(entry, remarks);
+ if (monitor != null) {
+ monitor.worked(1); // progress bar...
+ if (monitor.isCanceled()) {
+ stop_checking = true;
+ }
+ }
+ }
+ }
+ if (monitor != null) {
+ monitor.worked(1);
+ }
+ }
+ if (monitor != null) {
+ monitor.done();
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ private void compatibleEntryCheck(JobEntryInterface entry, List remarks) {
+ entry.check(remarks, this);
+ }
+
+ /**
+ * Gets the resource dependencies.
+ *
+ * @return the resource dependencies
+ */
+ public List getResourceDependencies() {
+ List resourceReferences = new ArrayList();
+ JobEntryCopy copy = null;
+ JobEntryInterface entry = null;
+ for (int i = 0; i < jobcopies.size(); i++) {
+ copy = jobcopies.get(i); // get the job entry copy
+ entry = copy.getEntry();
+ resourceReferences.addAll(entry.getResourceDependencies(this));
+ }
+
+ return resourceReferences;
+ }
+
+ public String exportResources(VariableSpace space, Map definitions,
+ ResourceNamingInterface namingInterface, Repository repository, IMetaStore metaStore) throws KettleException {
+ String resourceName = null;
+ try {
+ // Handle naming for both repository and XML bases resources...
+ //
+ String baseName;
+ String originalPath;
+ String fullname;
+ String extension = "kjb";
+ if (Const.isEmpty(getFilename())) {
+ // Assume repository...
+ //
+ originalPath = directory.getPath();
+ baseName = getName();
+ fullname =
+ directory.getPath() + (directory.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR) ? ""
+ : RepositoryDirectory.DIRECTORY_SEPARATOR) + getName() + "." + extension; //
+ } else {
+ // Assume file
+ //
+ FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(getFilename()), space);
+ originalPath = fileObject.getParent().getName().getPath();
+ baseName = fileObject.getName().getBaseName();
+ fullname = fileObject.getName().getPath();
+ }
+
+ resourceName = namingInterface
+ .nameResource(baseName, originalPath, extension, ResourceNamingInterface.FileNamingType.JOB);
+ ResourceDefinition definition = definitions.get(resourceName);
+ if (definition == null) {
+ // If we do this once, it will be plenty :-)
+ //
+ JobMeta jobMeta = (JobMeta) this.realClone(false);
+
+ // All objects get re-located to the root folder,
+ // but, when exporting, we need to see current directory
+ // in order to make 'Internal.Entry.Current.Directory' variable work
+ jobMeta.setRepositoryDirectory(directory);
+
+ // Add used resources, modify transMeta accordingly
+ // Go through the list of steps, etc.
+ // These critters change the steps in the cloned TransMeta
+ // At the end we make a new XML version of it in "exported"
+ // format...
+
+ // loop over steps, databases will be exported to XML anyway.
+ //
+ for (JobEntryCopy jobEntry : jobMeta.jobcopies) {
+ compatibleJobEntryExportResources(jobEntry.getEntry(), jobMeta, definitions, namingInterface, repository);
+ jobEntry.getEntry().exportResources(jobMeta, definitions, namingInterface, repository, metaStore);
+ }
+
+ // Set a number of parameters for all the data files referenced so far...
+ //
+ Map directoryMap = namingInterface.getDirectoryMap();
+ if (directoryMap != null) {
+ for (String directory : directoryMap.keySet()) {
+ String parameterName = directoryMap.get(directory);
+ jobMeta.addParameterDefinition(parameterName, directory, "Data file path discovered during export");
+ }
+ }
+
+ // At the end, add ourselves to the map...
+ //
+ String jobMetaContent = jobMeta.getXML();
+
+ definition = new ResourceDefinition(resourceName, jobMetaContent);
+
+ // Also remember the original filename (if any), including variables etc.
+ //
+ if (Const.isEmpty(this.getFilename())) { // Repository
+ definition.setOrigin(fullname);
+ } else {
+ definition.setOrigin(this.getFilename());
+ }
+
+ definitions.put(fullname, definition);
+ }
+ } catch (FileSystemException e) {
+ throw new KettleException(
+ BaseMessages.getString(PKG, "JobMeta.Exception.AnErrorOccuredReadingJob", getFilename()), e);
+ } catch (KettleFileException e) {
+ throw new KettleException(
+ BaseMessages.getString(PKG, "JobMeta.Exception.AnErrorOccuredReadingJob", getFilename()), e);
+ }
+
+ return resourceName;
+ }
+
+ @SuppressWarnings("deprecation")
+ private void compatibleJobEntryExportResources(JobEntryInterface entry, JobMeta jobMeta,
+ Map definitions, ResourceNamingInterface namingInterface, Repository repository2)
+ throws KettleException {
+ entry.exportResources(jobMeta, definitions, namingInterface, repository);
+ }
+
+ /**
+ * See if the name of the supplied job entry copy doesn't collide with any other job entry copy in the job.
+ *
+ * @param je The job entry copy to verify the name for.
+ */
+ public void renameJobEntryIfNameCollides(JobEntryCopy je) {
+ // First see if the name changed.
+ // If so, we need to verify that the name is not already used in the
+ // job.
+ //
+ String newname = je.getName();
+
+ // See if this name exists in the other job entries
+ //
+ boolean found;
+ int nr = 1;
+ do {
+ found = false;
+ for (JobEntryCopy copy : jobcopies) {
+ if (copy != je && copy.getName().equalsIgnoreCase(newname) && copy.getNr() == 0) {
+ found = true;
+ }
+ }
+ if (found) {
+ nr++;
+ newname = je.getName() + " (" + nr + ")";
+ }
+ } while (found);
+
+ // Rename if required.
+ //
+ je.setName(newname);
+ }
+
+ /**
+ * Gets the job copies.
+ *
+ * @return the job copies
+ */
+ public List getJobCopies() {
+ return jobcopies;
+ }
+
+ /**
+ * Gets the jobhops.
+ *
+ * @return the jobhops
+ */
+ public List getJobhops() {
+ return jobhops;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.repository.RepositoryElementInterface#getRepositoryElementType()
+ */
+ public RepositoryObjectType getRepositoryElementType() {
+ return REPOSITORY_ELEMENT_TYPE;
+ }
+
+ /**
+ * Create a unique list of job entry interfaces
+ *
+ * @return
+ */
+ public List composeJobEntryInterfaceList() {
+ List list = new ArrayList();
+
+ for (JobEntryCopy copy : jobcopies) {
+ if (!list.contains(copy.getEntry())) {
+ list.add(copy.getEntry());
+ }
+ }
+
+ return list;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.logging.LoggingObjectInterface#getLogChannelId()
+ */
+ public String getLogChannelId() {
+ return null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.pentaho.di.core.logging.LoggingObjectInterface#getObjectType()
+ */
+ public LoggingObjectType getObjectType() {
+ return LoggingObjectType.JOBMETA;
+ }
+
+ /**
+ * Gets the job entry log table.
+ *
+ * @return the jobEntryLogTable
+ */
+ public JobEntryLogTable getJobEntryLogTable() {
+ return jobEntryLogTable;
+ }
+
+ /**
+ * Sets the job entry log table.
+ *
+ * @param jobEntryLogTable the jobEntryLogTable to set
+ */
+ public void setJobEntryLogTable(JobEntryLogTable jobEntryLogTable) {
+ this.jobEntryLogTable = jobEntryLogTable;
+ }
+
+ /**
+ * Gets the log tables.
+ *
+ * @return the log tables
+ */
+ public List getLogTables() {
+ List logTables = new ArrayList();
+ logTables.add(jobLogTable);
+ logTables.add(jobEntryLogTable);
+ logTables.add(channelLogTable);
+ logTables.addAll(extraLogTables);
+ return logTables;
+ }
+
+ /**
+ * Checks whether the job has repository references.
+ *
+ * @return true if the job has repository references, false otherwise
+ */
+ public boolean hasRepositoryReferences() {
+ for (JobEntryCopy copy : jobcopies) {
+ if (copy.getEntry().hasRepositoryReferences()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Look up the references after import
+ *
+ * @param repository the repository to reference.
+ */
+ public void lookupRepositoryReferences(Repository repository) throws KettleException {
+ KettleException lastThrownException = null;
+ Map notFoundedReferences = new HashMap<>();
+ for (JobEntryCopy copy : jobcopies) {
+ if (copy.getEntry().hasRepositoryReferences()) {
+ try {
+ copy.getEntry().lookupRepositoryReferences(repository);
+ } catch (IdNotFoundException e) {
+ lastThrownException = e;
+ String path = e.getPathToObject();
+ String name = e.getObjectName();
+ String key = StringUtils.isEmpty(path) || path.equals("null") ? name : path + "/" + name;
+ notFoundedReferences.put(key, e.getObjectType());
+ }
+ }
+ }
+ if (lastThrownException != null && !notFoundedReferences.isEmpty()) {
+ throw new LookupReferencesException(lastThrownException, notFoundedReferences);
+ }
+ }
+
+ /**
+ * Returns whether or not the job is gathering metrics. For a JobMeta this is always false.
+ *
+ * @return is gathering metrics = false;
+ */
+ @Override
+ public boolean isGatheringMetrics() {
+ return false;
+ }
+
+ /**
+ * Sets whether or not the job is gathering metrics. This is a stub with not executable code.
+ */
+ @Override
+ public void setGatheringMetrics(boolean gatheringMetrics) {
+ }
+
+ @Override
+ public boolean isForcingSeparateLogging() {
+ return false;
+ }
+
+ @Override
+ public void setForcingSeparateLogging(boolean forcingSeparateLogging) {
+ }
+
+ /**
+ * This method needs to be called to store those objects which are used and referenced in the job metadata but not
+ * saved in the serialization.
+ *
+ * @param metaStore The store to save to
+ * @throws MetaStoreException in case there is an error.
+ */
+ public void saveMetaStoreObjects(Repository repository, IMetaStore metaStore) throws MetaStoreException {
+ }
+
+ public List getExtraLogTables() {
+ return extraLogTables;
+ }
+
+ public void setExtraLogTables(List extraLogTables) {
+ this.extraLogTables = extraLogTables;
+ }
+
+ public boolean containsJobCopy(JobEntryCopy jobCopy) {
+ return jobcopies.contains(jobCopy);
+ }
+
+ public List getMissingEntries() {
+ return missingEntries;
+ }
+
+ public void addMissingEntry(MissingEntry missingEntry) {
+ if (missingEntries == null) {
+ missingEntries = new ArrayList();
+ }
+ missingEntries.add(missingEntry);
+ }
+
+ public void removeMissingEntry(MissingEntry missingEntry) {
+ if (missingEntries != null && missingEntry != null && missingEntries.contains(missingEntry)) {
+ missingEntries.remove(missingEntry);
+ }
+ }
+
+ public boolean hasMissingPlugins() {
+ return missingEntries != null && !missingEntries.isEmpty();
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/RemoteJobEntryLogHelper.java b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/RemoteJobEntryLogHelper.java
new file mode 100644
index 0000000..a5d397f
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/RemoteJobEntryLogHelper.java
@@ -0,0 +1,95 @@
+/*! ******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+package org.pentaho.di.job.entries;
+
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.logging.LogChannelInterface;
+
+/**
+ * Utility class for streaming log entries from slave back to master.
+ *
+ * @author Zhichun Wu
+ */
+public final class RemoteJobEntryLogHelper {
+ private static final String UNKNOWN_SERVER = "unknown server";
+ private static final String UNKNOW_OBJECT = "unknown object";
+
+ private final LogChannelInterface logger;
+
+ private final String serverAddress;
+ private final String objectId;
+
+ private int lastLogEntryNo;
+
+ public RemoteJobEntryLogHelper(SlaveServer server, String objectId, LogChannelInterface logger) {
+ this.logger = logger;
+
+ this.serverAddress = server == null || server.getName() == null ? UNKNOWN_SERVER : server.getName();
+ this.objectId = objectId == null ? UNKNOW_OBJECT : objectId;
+
+ this.lastLogEntryNo = 0;
+ }
+
+ public int getLastLogEntryNo() {
+ return this.lastLogEntryNo;
+ }
+
+ public void log(String logString, int firstEntryLineNo, int lastEntryLineNo) {
+ if (logger == null || logString == null) {
+ return;
+ }
+
+ int length = logString.length();
+ int lineDiff = firstEntryLineNo - lastLogEntryNo;
+
+ if (length > 0 && lastLogEntryNo != lastEntryLineNo) {
+ try {
+ logger.logBasic(new StringBuilder()
+ .append("---> Replay logs L")
+ .append(firstEntryLineNo)
+ .append(" ~ L")
+ .append(lastEntryLineNo)
+ .append(" from [")
+ .append(objectId)
+ .append('@')
+ .append(serverAddress)
+ .append("]: ")
+ .append(length)
+ .append(" bytes <---").toString());
+
+ if (lineDiff != 0) {
+ logger.logError(new StringBuffer()
+ .append("*** Somehow we ")
+ .append(lineDiff > 0 ? "lost " : "got duplicated ")
+ .append(Math.abs(lineDiff))
+ .append(" lines of logs from [")
+ .append(objectId)
+ .append('@')
+ .append(serverAddress)
+ .append("] ***")
+ .toString());
+ }
+
+ logger.logBasic(logString);
+ } catch (Throwable t) {
+ // ignore as logging failure is trivial
+ // t.printStackTrace();
+ }
+ }
+
+ lastLogEntryNo = lastEntryLineNo;
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/job/JobEntryJob.java b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/job/JobEntryJob.java
new file mode 100644
index 0000000..f8d9390
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/job/JobEntryJob.java
@@ -0,0 +1,1547 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.job.entries.job;
+
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.*;
+import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.exception.KettleDatabaseException;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.exception.KettleXMLException;
+import org.pentaho.di.core.logging.LogChannelFileWriter;
+import org.pentaho.di.core.logging.LogLevel;
+import org.pentaho.di.core.parameters.DuplicateParamException;
+import org.pentaho.di.core.parameters.NamedParams;
+import org.pentaho.di.core.parameters.NamedParamsDefault;
+import org.pentaho.di.core.util.CurrentDirectoryResolver;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.variables.Variables;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.core.xml.XMLHandler;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.DelegationListener;
+import org.pentaho.di.job.Job;
+import org.pentaho.di.job.JobExecutionConfiguration;
+import org.pentaho.di.job.JobMeta;
+import org.pentaho.di.job.entries.RemoteJobEntryLogHelper;
+import org.pentaho.di.job.entry.JobEntryBase;
+import org.pentaho.di.job.entry.JobEntryInterface;
+import org.pentaho.di.job.entry.validator.AndValidator;
+import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;
+import org.pentaho.di.repository.*;
+import org.pentaho.di.resource.*;
+import org.pentaho.di.resource.ResourceEntry.ResourceType;
+import org.pentaho.di.www.SlaveServerJobStatus;
+import org.pentaho.metastore.api.IMetaStore;
+import org.w3c.dom.Node;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+/**
+ * Recursive definition of a Job. This step means that an entire Job has to be executed. It can be the same Job, but
+ * just make sure that you don't get an endless loop. Provide an escape routine using JobEval.
+ *
+ * @author Matt
+ * @since 01-10-2003, Rewritten on 18-06-2004
+ */
+public class JobEntryJob extends JobEntryBase implements Cloneable, JobEntryInterface {
+ private static Class PKG = JobEntryJob.class; // for i18n purposes, needed by Translator2!!
+
+ private String filename;
+ private String jobname;
+ private String directory;
+ private ObjectId jobObjectId;
+ private ObjectLocationSpecificationMethod specificationMethod;
+
+ public String[] arguments;
+ public boolean argFromPrevious;
+ public boolean paramsFromPrevious;
+ public boolean execPerRow;
+
+ public String[] parameters;
+ public String[] parameterFieldNames;
+ public String[] parameterValues;
+
+ public boolean setLogfile;
+ public String logfile, logext;
+ public boolean addDate, addTime;
+ public LogLevel logFileLevel;
+
+ public boolean parallel;
+ private String directoryPath;
+ public boolean setAppendLogfile;
+ public boolean createParentFolder;
+
+ public boolean waitingToFinish = true;
+ public boolean followingAbortRemotely;
+
+ public boolean expandingRemoteJob;
+
+ private String remoteSlaveServerName;
+ public boolean passingAllParameters = true;
+
+ private boolean passingExport;
+
+ public static final LogLevel DEFAULT_LOG_LEVEL = LogLevel.NOTHING;
+
+ private Job job;
+
+ public JobEntryJob(String name) {
+ super(name, "");
+ }
+
+ public JobEntryJob() {
+ this("");
+ clear();
+ }
+
+ private void allocateArgs(int nrArgs) {
+ arguments = new String[nrArgs];
+ }
+
+ private void allocateParams(int nrParameters) {
+ parameters = new String[nrParameters];
+ parameterFieldNames = new String[nrParameters];
+ parameterValues = new String[nrParameters];
+ }
+
+ public Object clone() {
+ JobEntryJob je = (JobEntryJob) super.clone();
+ if (arguments != null) {
+ int nrArgs = arguments.length;
+ je.allocateArgs(nrArgs);
+ System.arraycopy(arguments, 0, je.arguments, 0, nrArgs);
+ }
+ if (parameters != null) {
+ int nrParameters = parameters.length;
+ je.allocateParams(nrParameters);
+ System.arraycopy(parameters, 0, je.parameters, 0, nrParameters);
+ System.arraycopy(parameterFieldNames, 0, je.parameterFieldNames, 0, nrParameters);
+ System.arraycopy(parameterValues, 0, je.parameterValues, 0, nrParameters);
+ }
+ return je;
+ }
+
+ public void setFileName(String n) {
+ filename = n;
+ }
+
+ /**
+ * @return the filename
+ * @deprecated use getFilename() instead.
+ */
+ @Deprecated
+ public String getFileName() {
+ return filename;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+
+ public String getRealFilename() {
+ return environmentSubstitute(getFilename());
+ }
+
+ public void setJobName(String jobname) {
+ this.jobname = jobname;
+ }
+
+ public String getJobName() {
+ return jobname;
+ }
+
+ public String getDirectory() {
+ return directory;
+ }
+
+ public void setDirectory(String directory) {
+ this.directory = directory;
+ }
+
+ public boolean isPassingExport() {
+ return passingExport;
+ }
+
+ public void setPassingExport(boolean passingExport) {
+ this.passingExport = passingExport;
+ }
+
+ public String getLogFilename() {
+ String retval = "";
+ if (setLogfile) {
+ retval += logfile == null ? "" : logfile;
+ Calendar cal = Calendar.getInstance();
+ if (addDate) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ retval += "_" + sdf.format(cal.getTime());
+ }
+ if (addTime) {
+ SimpleDateFormat sdf = new SimpleDateFormat("HHmmss");
+ retval += "_" + sdf.format(cal.getTime());
+ }
+ if (logext != null && logext.length() > 0) {
+ retval += "." + logext;
+ }
+ }
+ return retval;
+ }
+
+ public String getXML() {
+ StringBuffer retval = new StringBuffer(200);
+
+ retval.append(super.getXML());
+
+ // specificationMethod
+ //
+ retval.append(" ").append(
+ XMLHandler.addTagValue("specification_method", specificationMethod == null ? null : specificationMethod
+ .getCode()));
+ retval.append(" ").append(
+ XMLHandler.addTagValue("job_object_id", jobObjectId == null ? null : jobObjectId.toString()));
+ // Export a little bit of extra information regarding the reference since it doesn't really matter outside the same
+ // repository.
+ //
+ if (rep != null && jobObjectId != null) {
+ try {
+ RepositoryObject objectInformation = rep.getObjectInformation(jobObjectId, RepositoryObjectType.JOB);
+ if (objectInformation != null) {
+ jobname = objectInformation.getName();
+ directory = objectInformation.getRepositoryDirectory().getPath();
+ }
+ } catch (KettleException e) {
+ // Ignore object reference problems. It simply means that the reference is no longer valid.
+ }
+ }
+ retval.append(" ").append(XMLHandler.addTagValue("filename", filename));
+ retval.append(" ").append(XMLHandler.addTagValue("jobname", jobname));
+
+ if (directory != null) {
+ retval.append(" ").append(XMLHandler.addTagValue("directory", directory));
+ } else if (directoryPath != null) {
+ retval.append(" ").append(XMLHandler.addTagValue("directory", directoryPath));
+ }
+ retval.append(" ").append(XMLHandler.addTagValue("arg_from_previous", argFromPrevious));
+ retval.append(" ").append(XMLHandler.addTagValue("params_from_previous", paramsFromPrevious));
+ retval.append(" ").append(XMLHandler.addTagValue("exec_per_row", execPerRow));
+ retval.append(" ").append(XMLHandler.addTagValue("set_logfile", setLogfile));
+ retval.append(" ").append(XMLHandler.addTagValue("logfile", logfile));
+ retval.append(" ").append(XMLHandler.addTagValue("logext", logext));
+ retval.append(" ").append(XMLHandler.addTagValue("add_date", addDate));
+ retval.append(" ").append(XMLHandler.addTagValue("add_time", addTime));
+ retval.append(" ").append(
+ XMLHandler.addTagValue("loglevel", logFileLevel != null ? logFileLevel.getCode() : DEFAULT_LOG_LEVEL
+ .getCode()));
+ retval.append(" ").append(XMLHandler.addTagValue("slave_server_name", remoteSlaveServerName));
+ retval.append(" ").append(XMLHandler.addTagValue("wait_until_finished", waitingToFinish));
+ retval.append(" ").append(XMLHandler.addTagValue("follow_abort_remote", followingAbortRemotely));
+ retval.append(" ").append(XMLHandler.addTagValue("expand_remote_job", expandingRemoteJob));
+ retval.append(" ").append(XMLHandler.addTagValue("create_parent_folder", createParentFolder));
+ retval.append(" ").append(XMLHandler.addTagValue("pass_export", passingExport));
+
+ if (arguments != null) {
+ for (int i = 0; i < arguments.length; i++) {
+ // This is a very very bad way of making an XML file, don't use it (or
+ // copy it). Sven Boden
+ retval.append(" ").append(XMLHandler.addTagValue("argument" + i, arguments[i]));
+ }
+ }
+
+ if (parameters != null) {
+ retval.append(" ").append(XMLHandler.openTag("parameters"));
+
+ retval.append(" ").append(XMLHandler.addTagValue("pass_all_parameters", passingAllParameters));
+
+ for (int i = 0; i < parameters.length; i++) {
+ // This is a better way of making the XML file than the arguments.
+ retval.append(" ").append(XMLHandler.openTag("parameter"));
+
+ retval.append(" ").append(XMLHandler.addTagValue("name", parameters[i]));
+ retval.append(" ").append(XMLHandler.addTagValue("stream_name", parameterFieldNames[i]));
+ retval.append(" ").append(XMLHandler.addTagValue("value", parameterValues[i]));
+
+ retval.append(" ").append(XMLHandler.closeTag("parameter"));
+ }
+ retval.append(" ").append(XMLHandler.closeTag("parameters"));
+ }
+ retval.append(" ").append(XMLHandler.addTagValue("set_append_logfile", setAppendLogfile));
+
+ return retval.toString();
+ }
+
+ private void checkObjectLocationSpecificationMethod() {
+ if (specificationMethod == null) {
+ // Backward compatibility
+ //
+ // Default = Filename
+ //
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+
+ if (!Const.isEmpty(filename)) {
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+ } else if (jobObjectId != null) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_REFERENCE;
+ } else if (!Const.isEmpty(jobname)) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_NAME;
+ }
+ }
+ }
+
+ public void loadXML(Node entrynode, List databases, List slaveServers,
+ Repository rep, IMetaStore metaStore) throws KettleXMLException {
+ try {
+ super.loadXML(entrynode, databases, slaveServers);
+
+ String method = XMLHandler.getTagValue(entrynode, "specification_method");
+ specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
+
+ String jobId = XMLHandler.getTagValue(entrynode, "job_object_id");
+ jobObjectId = Const.isEmpty(jobId) ? null : new StringObjectId(jobId);
+ filename = XMLHandler.getTagValue(entrynode, "filename");
+ jobname = XMLHandler.getTagValue(entrynode, "jobname");
+
+ if (rep != null && rep.isConnected() && !Const.isEmpty(jobname)) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_NAME;
+ }
+
+ // Backward compatibility check for object specification
+ //
+ checkObjectLocationSpecificationMethod();
+
+ argFromPrevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "arg_from_previous"));
+ paramsFromPrevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "params_from_previous"));
+ execPerRow = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "exec_per_row"));
+ setLogfile = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "set_logfile"));
+ addDate = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "add_date"));
+ addTime = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "add_time"));
+ logfile = XMLHandler.getTagValue(entrynode, "logfile");
+ logext = XMLHandler.getTagValue(entrynode, "logext");
+ logFileLevel = LogLevel.getLogLevelForCode(XMLHandler.getTagValue(entrynode, "loglevel"));
+ setAppendLogfile = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "set_append_logfile"));
+ remoteSlaveServerName = XMLHandler.getTagValue(entrynode, "slave_server_name");
+ passingExport = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "pass_export"));
+ directory = XMLHandler.getTagValue(entrynode, "directory");
+ createParentFolder = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "create_parent_folder"));
+
+ String wait = XMLHandler.getTagValue(entrynode, "wait_until_finished");
+ if (Const.isEmpty(wait)) {
+ waitingToFinish = true;
+ } else {
+ waitingToFinish = "Y".equalsIgnoreCase(wait);
+ }
+
+ followingAbortRemotely = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "follow_abort_remote"));
+ expandingRemoteJob = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "expand_remote_job"));
+
+ // How many arguments?
+ int argnr = 0;
+ while (XMLHandler.getTagValue(entrynode, "argument" + argnr) != null) {
+ argnr++;
+ }
+ allocateArgs(argnr);
+
+ // Read them all... This is a very BAD way to do it by the way. Sven
+ // Boden.
+ for (int a = 0; a < argnr; a++) {
+ arguments[a] = XMLHandler.getTagValue(entrynode, "argument" + a);
+ }
+
+ Node parametersNode = XMLHandler.getSubNode(entrynode, "parameters");
+
+ String passAll = XMLHandler.getTagValue(parametersNode, "pass_all_parameters");
+ passingAllParameters = Const.isEmpty(passAll) || "Y".equalsIgnoreCase(passAll);
+
+ int nrParameters = XMLHandler.countNodes(parametersNode, "parameter");
+ allocateParams(nrParameters);
+
+ for (int i = 0; i < nrParameters; i++) {
+ Node knode = XMLHandler.getSubNodeByNr(parametersNode, "parameter", i);
+
+ parameters[i] = XMLHandler.getTagValue(knode, "name");
+ parameterFieldNames[i] = XMLHandler.getTagValue(knode, "stream_name");
+ parameterValues[i] = XMLHandler.getTagValue(knode, "value");
+ }
+ } catch (KettleXMLException xe) {
+ throw new KettleXMLException("Unable to load 'job' job entry from XML node", xe);
+ }
+ }
+
+ /**
+ * Load the jobentry from repository
+ */
+ public void loadRep(Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List databases,
+ List slaveServers) throws KettleException {
+ try {
+ String method = rep.getJobEntryAttributeString(id_jobentry, "specification_method");
+ specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
+ String jobId = rep.getJobEntryAttributeString(id_jobentry, "job_object_id");
+ jobObjectId = Const.isEmpty(jobId) ? null : new StringObjectId(jobId);
+ jobname = rep.getJobEntryAttributeString(id_jobentry, "name");
+ directory = rep.getJobEntryAttributeString(id_jobentry, "dir_path");
+ filename = rep.getJobEntryAttributeString(id_jobentry, "file_name");
+
+ // Backward compatibility check for object specification
+ //
+ checkObjectLocationSpecificationMethod();
+
+ argFromPrevious = rep.getJobEntryAttributeBoolean(id_jobentry, "arg_from_previous");
+ paramsFromPrevious = rep.getJobEntryAttributeBoolean(id_jobentry, "params_from_previous");
+ execPerRow = rep.getJobEntryAttributeBoolean(id_jobentry, "exec_per_row");
+ setLogfile = rep.getJobEntryAttributeBoolean(id_jobentry, "set_logfile");
+ addDate = rep.getJobEntryAttributeBoolean(id_jobentry, "add_date");
+ addTime = rep.getJobEntryAttributeBoolean(id_jobentry, "add_time");
+ logfile = rep.getJobEntryAttributeString(id_jobentry, "logfile");
+ logext = rep.getJobEntryAttributeString(id_jobentry, "logext");
+ logFileLevel = LogLevel.getLogLevelForCode(rep.getJobEntryAttributeString(id_jobentry, "loglevel"));
+ setAppendLogfile = rep.getJobEntryAttributeBoolean(id_jobentry, "set_append_logfile");
+ remoteSlaveServerName = rep.getJobEntryAttributeString(id_jobentry, "slave_server_name");
+ passingExport = rep.getJobEntryAttributeBoolean(id_jobentry, "pass_export");
+ waitingToFinish = rep.getJobEntryAttributeBoolean(id_jobentry, "wait_until_finished", true);
+ followingAbortRemotely = rep.getJobEntryAttributeBoolean(id_jobentry, "follow_abort_remote");
+ expandingRemoteJob = rep.getJobEntryAttributeBoolean(id_jobentry, "expand_remote_job");
+ createParentFolder = rep.getJobEntryAttributeBoolean(id_jobentry, "create_parent_folder");
+
+ // How many arguments?
+ int argnr = rep.countNrJobEntryAttributes(id_jobentry, "argument");
+ allocateArgs(argnr);
+
+ // Read all arguments ...
+ for (int a = 0; a < argnr; a++) {
+ arguments[a] = rep.getJobEntryAttributeString(id_jobentry, a, "argument");
+ }
+
+ // How many arguments?
+ int parameternr = rep.countNrJobEntryAttributes(id_jobentry, "parameter_name");
+ allocateParams(parameternr);
+
+ // Read all parameters ...
+ for (int a = 0; a < parameternr; a++) {
+ parameters[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_name");
+ parameterFieldNames[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_stream_name");
+ parameterValues[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_value");
+ }
+
+ passingAllParameters = rep.getJobEntryAttributeBoolean(id_jobentry, "pass_all_parameters", true);
+
+ } catch (KettleDatabaseException dbe) {
+ throw new KettleException("Unable to load job entry of type 'job' from the repository with id_jobentry="
+ + id_jobentry, dbe);
+ }
+ }
+
+ // Save the attributes of this job entry
+ //
+ public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_job) throws KettleException {
+ try {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "specification_method", specificationMethod == null
+ ? null : specificationMethod.getCode());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "job_object_id", jobObjectId == null ? null : jobObjectId
+ .toString());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "name", getJobName());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "dir_path", getDirectory() != null ? getDirectory() : "");
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "file_name", filename);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "arg_from_previous", argFromPrevious);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "params_from_previous", paramsFromPrevious);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "exec_per_row", execPerRow);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "set_logfile", setLogfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "add_date", addDate);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "add_time", addTime);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "logfile", logfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "logext", logext);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "set_append_logfile", setAppendLogfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "loglevel", logFileLevel != null
+ ? logFileLevel.getCode() : JobEntryJob.DEFAULT_LOG_LEVEL.getCode());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "slave_server_name", remoteSlaveServerName);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "pass_export", passingExport);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "wait_until_finished", waitingToFinish);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "follow_abort_remote", followingAbortRemotely);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "expand_remote_job", expandingRemoteJob);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "create_parent_folder", createParentFolder);
+
+ // save the arguments...
+ if (arguments != null) {
+ for (int i = 0; i < arguments.length; i++) {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "argument", arguments[i]);
+ }
+ }
+
+ // save the parameters...
+ if (parameters != null) {
+ for (int i = 0; i < parameters.length; i++) {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_name", parameters[i]);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_stream_name", Const.NVL(
+ parameterFieldNames[i], ""));
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_value", Const.NVL(
+ parameterValues[i], ""));
+ }
+ }
+
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "pass_all_parameters", passingAllParameters);
+ } catch (KettleDatabaseException dbe) {
+ throw new KettleException(
+ "Unable to save job entry of type job to the repository with id_job=" + id_job, dbe);
+ }
+ }
+
+ public Result execute(Result result, int nr) throws KettleException {
+ result.setEntryNr(nr);
+
+ LogChannelFileWriter logChannelFileWriter = null;
+
+ LogLevel jobLogLevel = parentJob.getLogLevel();
+ if (setLogfile) {
+ String realLogFilename = environmentSubstitute(getLogFilename());
+ // We need to check here the log filename
+ // if we do not have one, we must fail
+ if (Const.isEmpty(realLogFilename)) {
+ logError(BaseMessages.getString(PKG, "JobJob.Exception.LogFilenameMissing"));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+
+ // create parent folder?
+ if (!createParentFolder(realLogFilename)) {
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ try {
+ logChannelFileWriter =
+ new LogChannelFileWriter(
+ this.getLogChannelId(), KettleVFS.getFileObject(realLogFilename), setAppendLogfile);
+ logChannelFileWriter.startLogging();
+ } catch (KettleException e) {
+ logError("Unable to open file appender for file [" + getLogFilename() + "] : " + e.toString());
+ logError(Const.getStackTracker(e));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ jobLogLevel = logFileLevel;
+ }
+
+ // Figure out the remote slave server...
+ //
+ SlaveServer remoteSlaveServer = null;
+ if (!Const.isEmpty(remoteSlaveServerName)) {
+ String realRemoteSlaveServerName = environmentSubstitute(remoteSlaveServerName);
+ remoteSlaveServer = parentJob.getJobMeta().findSlaveServer(realRemoteSlaveServerName);
+ if (remoteSlaveServer == null) {
+ throw new KettleException(BaseMessages.getString(
+ PKG, "JobJob.Exception.UnableToFindRemoteSlaveServer", realRemoteSlaveServerName));
+ }
+ }
+ try {
+ // First load the job, outside of the loop...
+ if (parentJob.getJobMeta() != null) {
+ // reset the internal variables again.
+ // Maybe we should split up the variables even more like in UNIX shells.
+ // The internal variables need to be reset to be able use them properly
+ // in 2 sequential sub jobs.
+ parentJob.getJobMeta().setInternalKettleVariables();
+ }
+
+ // Explain what we are loading...
+ //
+ switch (specificationMethod) {
+ case REPOSITORY_BY_NAME:
+ if (log.isDetailed()) {
+ logDetailed("Loading job from repository : ["
+ + directory + " : " + environmentSubstitute(jobname) + "]");
+ }
+ break;
+ case FILENAME:
+ if (log.isDetailed()) {
+ logDetailed("Loading job from XML file : [" + environmentSubstitute(filename) + "]");
+ }
+ break;
+ case REPOSITORY_BY_REFERENCE:
+ if (log.isDetailed()) {
+ logDetailed("Loading job from repository by reference : [" + jobObjectId + "]");
+ }
+ break;
+ default:
+ break;
+ }
+
+ JobMeta jobMeta = getJobMeta(rep, this);
+
+ // Verify that we loaded something, complain if we did not...
+ //
+ if (jobMeta == null) {
+ throw new KettleException(
+ "Unable to load the job: please specify the name and repository directory OR a filename");
+ }
+
+ verifyRecursiveExecution(parentJob, jobMeta);
+
+ int iteration = 0;
+ String[] args1 = arguments;
+ // no arguments? Check the parent jobs arguments
+ if (args1 == null || args1.length == 0) {
+ args1 = parentJob.getArguments();
+ }
+
+ copyVariablesFrom(parentJob);
+ setParentVariableSpace(parentJob);
+
+ //
+ // For the moment only do variable translation at the start of a job, not
+ // for every input row (if that would be switched on)
+ //
+ String[] args = null;
+ if (args1 != null) {
+ args = new String[args1.length];
+ for (int idx = 0; idx < args1.length; idx++) {
+ args[idx] = environmentSubstitute(args1[idx]);
+ }
+ }
+
+ RowMetaAndData resultRow = null;
+ boolean first = true;
+ List rows = new ArrayList(result.getRows());
+
+ while ((first && !execPerRow)
+ || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
+ first = false;
+
+ // Clear the result rows of the result
+ // Otherwise we double the amount of rows every iteration in the simple cases.
+ //
+ if (execPerRow) {
+ result.getRows().clear();
+ }
+
+ if (rows != null && execPerRow) {
+ resultRow = rows.get(iteration);
+ } else {
+ resultRow = null;
+ }
+
+ NamedParams namedParam = new NamedParamsDefault();
+
+ // First (optionally) copy all the parameter values from the parent job
+ //
+ if (paramsFromPrevious) {
+ String[] parentParameters = parentJob.listParameters();
+ for (int idx = 0; idx < parentParameters.length; idx++) {
+ String par = parentParameters[idx];
+ String def = parentJob.getParameterDefault(par);
+ String val = parentJob.getParameterValue(par);
+ String des = parentJob.getParameterDescription(par);
+
+ namedParam.addParameterDefinition(par, def, des);
+ namedParam.setParameterValue(par, val);
+ }
+ }
+
+ // Now add those parameter values specified by the user in the job entry
+ //
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+
+ // If it's not yet present in the parent job, add it...
+ //
+ if (Const.indexOfString(parameters[idx], namedParam.listParameters()) < 0) {
+ // We have a parameter
+ try {
+ namedParam.addParameterDefinition(parameters[idx], "", "Job entry runtime");
+ } catch (DuplicateParamException e) {
+ // Should never happen
+ //
+ logError("Duplicate parameter definition for " + parameters[idx]);
+ }
+ }
+
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ namedParam.setParameterValue(parameters[idx], Const.NVL(
+ environmentSubstitute(parameterValues[idx]), ""));
+ } else {
+ // something filled in, in the field column...
+ //
+ String value = "";
+ if (resultRow != null) {
+ value = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ namedParam.setParameterValue(parameters[idx], value);
+ }
+ }
+ }
+ }
+
+ Result oneResult = new Result();
+
+ List sourceRows = null;
+
+ if (execPerRow) {
+ // Execute for each input row
+
+ if (argFromPrevious) {
+ // Copy the input row to the (command line) arguments
+
+ args = null;
+ if (resultRow != null) {
+ args = new String[resultRow.size()];
+ for (int i = 0; i < resultRow.size(); i++) {
+ args[i] = resultRow.getString(i, null);
+ }
+ }
+ } else {
+ // Just pass a single row
+ List newList = new ArrayList();
+ newList.add(resultRow);
+ sourceRows = newList;
+ }
+
+ if (paramsFromPrevious) { // Copy the input the parameters
+
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+ // We have a parameter
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ namedParam.setParameterValue(parameters[idx], Const.NVL(
+ environmentSubstitute(parameterValues[idx]), ""));
+ } else {
+ String fieldValue = "";
+
+ if (resultRow != null) {
+ fieldValue = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ // Get the value from the input stream
+ namedParam.setParameterValue(parameters[idx], Const.NVL(fieldValue, ""));
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (argFromPrevious) {
+ // Only put the first Row on the arguments
+ args = null;
+ if (resultRow != null) {
+ args = new String[resultRow.size()];
+ for (int i = 0; i < resultRow.size(); i++) {
+ args[i] = resultRow.getString(i, null);
+ }
+ }
+ } else {
+ // Keep it as it was...
+ sourceRows = result.getRows();
+ }
+
+ if (paramsFromPrevious) { // Copy the input the parameters
+
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+ // We have a parameter
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ namedParam.setParameterValue(parameters[idx], Const.NVL(
+ environmentSubstitute(parameterValues[idx]), ""));
+ } else {
+ String fieldValue = "";
+
+ if (resultRow != null) {
+ fieldValue = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ // Get the value from the input stream
+ namedParam.setParameterValue(parameters[idx], Const.NVL(fieldValue, ""));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (remoteSlaveServer == null) {
+ // Local execution...
+ //
+
+ // Create a new job
+ //
+ job = new Job(rep, jobMeta, this);
+ job.setParentJob(parentJob);
+ job.setLogLevel(jobLogLevel);
+ job.shareVariablesWith(this);
+ job.setInternalKettleVariables(this);
+ job.copyParametersFrom(jobMeta);
+ job.setInteractive(parentJob.isInteractive());
+ if (job.isInteractive()) {
+ job.getJobEntryListeners().addAll(parentJob.getJobEntryListeners());
+ }
+
+ // Pass the socket repository all around.
+ //
+ job.setSocketRepository(parentJob.getSocketRepository());
+
+ // Set the parameters calculated above on this instance.
+ //
+ job.clearParameters();
+ String[] parameterNames = job.listParameters();
+ for (int idx = 0; idx < parameterNames.length; idx++) {
+ // Grab the parameter value set in the job entry
+ //
+ String thisValue = namedParam.getParameterValue(parameterNames[idx]);
+ if (!Const.isEmpty(thisValue)) {
+ // Set the value as specified by the user in the job entry
+ //
+ job.setParameterValue(parameterNames[idx], thisValue);
+ } else {
+ // See if the parameter had a value set in the parent job...
+ // This value should pass down to the sub-job if that's what we
+ // opted to do.
+ //
+ if (isPassingAllParameters()) {
+ String parentValue = parentJob.getParameterValue(parameterNames[idx]);
+ if (!Const.isEmpty(parentValue)) {
+ job.setParameterValue(parameterNames[idx], parentValue);
+ }
+ }
+ }
+ }
+ job.activateParameters();
+
+ // Set the source rows we calculated above...
+ //
+ job.setSourceRows(sourceRows);
+
+ // Don't forget the logging...
+ job.beginProcessing();
+
+ // Link the job with the sub-job
+ parentJob.getJobTracker().addJobTracker(job.getJobTracker());
+
+ // Link both ways!
+ job.getJobTracker().setParentJobTracker(parentJob.getJobTracker());
+
+ if (parentJob.getJobMeta().isBatchIdPassed()) {
+ job.setPassedBatchId(parentJob.getBatchId());
+ }
+
+ job.setArguments(args);
+
+ // Inform the parent job we started something here...
+ //
+ for (DelegationListener delegationListener : parentJob.getDelegationListeners()) {
+ // TODO: copy some settings in the job execution configuration, not strictly needed
+ // but the execution configuration information is useful in case of a job re-start
+ //
+ delegationListener.jobDelegationStarted(job, new JobExecutionConfiguration());
+ }
+
+ JobEntryJobRunner runner = new JobEntryJobRunner(job, result, nr, log);
+ Thread jobRunnerThread = new Thread(runner);
+ // PDI-6518
+ // added UUID to thread name, otherwise threads do share names if jobs entries are executed in parallel in a
+ // parent job
+ // if that happens, contained transformations start closing each other's connections
+ jobRunnerThread.setName(Const.NVL(job.getJobMeta().getName(), job.getJobMeta().getFilename())
+ + " UUID: " + UUID.randomUUID().toString());
+ jobRunnerThread.start();
+
+ // Keep running until we're done.
+ //
+ while (!runner.isFinished() && !parentJob.isStopped()) {
+ try {
+ Thread.sleep(0, 1);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+
+ // if the parent-job was stopped, stop the sub-job too...
+ if (parentJob.isStopped()) {
+ job.stopAll();
+ runner.waitUntilFinished(); // Wait until finished!
+ }
+
+ oneResult = runner.getResult();
+
+ } else {
+
+ // Make sure we can parameterize the slave server connection
+ //
+ remoteSlaveServer.shareVariablesWith(this);
+
+ // Remote execution...
+ //
+ JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration();
+ jobExecutionConfiguration.setPreviousResult(result.lightClone()); // lightClone() because rows are
+ // overwritten in next line.
+ jobExecutionConfiguration.getPreviousResult().setRows(sourceRows);
+ jobExecutionConfiguration.setArgumentStrings(args);
+ jobExecutionConfiguration.setVariables(this);
+ jobExecutionConfiguration.setRemoteServer(remoteSlaveServer);
+ jobExecutionConfiguration.setRepository(rep);
+ jobExecutionConfiguration.setLogLevel(jobLogLevel);
+ jobExecutionConfiguration.setPassingExport(passingExport);
+ jobExecutionConfiguration.setExpandingRemoteJob(expandingRemoteJob);
+ for (String param : namedParam.listParameters()) {
+ String defValue = namedParam.getParameterDefault(param);
+ String value = namedParam.getParameterValue(param);
+ jobExecutionConfiguration.getParams().put(param, Const.NVL(value, defValue));
+ }
+ if (parentJob.getJobMeta().isBatchIdPassed()) {
+ jobExecutionConfiguration.setPassedBatchId(parentJob.getBatchId());
+ }
+
+ // Send the XML over to the slave server
+ // Also start the job over there...
+ //
+ String carteObjectId = null;
+ try {
+ carteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, rep, metaStore);
+ } catch (KettleException e) {
+ // Perhaps the job exists on the remote server, carte is down, etc.
+ // This is an abort situation, stop the parent job...
+ // We want this in case we are running in parallel. The other job
+ // entries can stop running now.
+ //
+ parentJob.stopAll();
+
+ // Pass the exception along
+ //
+ throw e;
+ }
+
+ // Now start the monitoring...
+ //
+ SlaveServerJobStatus jobStatus = null;
+
+ RemoteJobEntryLogHelper logHelper = new RemoteJobEntryLogHelper(
+ remoteSlaveServer, carteObjectId, this.log);
+
+ while (!parentJob.isStopped() && waitingToFinish) {
+ try {
+ jobStatus = remoteSlaveServer.getJobStatus(jobMeta.getName(), carteObjectId,
+ logHelper.getLastLogEntryNo());
+ logHelper.log(jobStatus.getLoggingString(),
+ jobStatus.getFirstLoggingLineNr(), jobStatus.getLastLoggingLineNr());
+
+
+ if (jobStatus.getResult() != null) {
+ // The job is finished, get the result...
+ //
+ oneResult = jobStatus.getResult();
+ break;
+ }
+ } catch (Exception e1) {
+ logError("Unable to contact slave server ["
+ + remoteSlaveServer + "] to verify the status of job [" + jobMeta.getName() + "]", e1);
+ oneResult.setNrErrors(1L);
+ break; // Stop looking too, chances are too low the server will
+ // come back on-line
+ }
+
+ // FIXME why the hell JobEntryTrans sleeps two seconds instead of one
+ // sleep for 1 second
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+
+ if (!waitingToFinish) {
+ // Since the job was posted successfully, the result is true...
+ //
+ oneResult = new Result();
+ oneResult.setResult(true);
+ }
+
+ if (parentJob.isStopped()) {
+ try {
+ // See if we have a status and if we need to stop the remote
+ // execution here...
+ //
+ if (jobStatus == null || jobStatus.isRunning()) {
+ // Try a remote abort ...
+ //
+ remoteSlaveServer.stopJob(jobMeta.getName(), carteObjectId);
+ }
+ } catch (Exception e1) {
+ logError("Unable to contact slave server ["
+ + remoteSlaveServer + "] to stop job [" + jobMeta.getName() + "]", e1);
+ oneResult.setNrErrors(1L);
+ break; // Stop looking too, chances are too low the server will
+ // come back on-line
+ }
+ }
+
+ }
+
+ result.clear(); // clear only the numbers, NOT the files or rows.
+ result.add(oneResult);
+
+ // Set the result rows too, if any ...
+ if (!Const.isEmpty(oneResult.getRows())) {
+ result.setRows(new ArrayList(oneResult.getRows()));
+ }
+
+ // if one of them fails (in the loop), increase the number of errors
+ //
+ if (oneResult.getResult() == false) {
+ result.setNrErrors(result.getNrErrors() + 1);
+ }
+
+ iteration++;
+ }
+
+ } catch (KettleException ke) {
+ logError("Error running job entry 'job' : ", ke);
+
+ result.setResult(false);
+ result.setNrErrors(1L);
+ }
+
+ if (setLogfile) {
+ if (logChannelFileWriter != null) {
+ logChannelFileWriter.stopLogging();
+
+ ResultFile resultFile =
+ new ResultFile(
+ ResultFile.FILE_TYPE_LOG, logChannelFileWriter.getLogFile(), parentJob.getJobname(), getName());
+ result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
+
+ // See if anything went wrong during file writing...
+ //
+ if (logChannelFileWriter.getException() != null) {
+ logError("Unable to open log file [" + getLogFilename() + "] : ");
+ logError(Const.getStackTracker(logChannelFileWriter.getException()));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ }
+ }
+
+ if (result.getNrErrors() > 0) {
+ result.setResult(false);
+ } else {
+ result.setResult(true);
+ }
+
+ return result;
+ }
+
+ private boolean createParentFolder(String filename) {
+ // Check for parent folder
+ FileObject parentfolder = null;
+ boolean resultat = true;
+ try {
+ // Get parent folder
+ parentfolder = KettleVFS.getFileObject(filename, this).getParent();
+ if (!parentfolder.exists()) {
+ if (createParentFolder) {
+ if (log.isDebug()) {
+ log.logDebug(BaseMessages.getString(PKG, "JobJob.Log.ParentLogFolderNotExist", parentfolder
+ .getName().toString()));
+ }
+ parentfolder.createFolder();
+ if (log.isDebug()) {
+ log.logDebug(BaseMessages.getString(PKG, "JobJob.Log.ParentLogFolderCreated", parentfolder
+ .getName().toString()));
+ }
+ } else {
+ log.logError(BaseMessages.getString(PKG, "JobJob.Log.ParentLogFolderNotExist", parentfolder
+ .getName().toString()));
+ resultat = false;
+ }
+ } else {
+ if (log.isDebug()) {
+ log.logDebug(BaseMessages.getString(PKG, "JobJob.Log.ParentLogFolderExists", parentfolder
+ .getName().toString()));
+ }
+ }
+ } catch (Exception e) {
+ resultat = false;
+ log.logError(BaseMessages.getString(PKG, "JobJob.Error.ChekingParentLogFolderTitle"), BaseMessages
+ .getString(PKG, "JobJob.Error.ChekingParentLogFolder", parentfolder.getName().toString()), e);
+ } finally {
+ if (parentfolder != null) {
+ try {
+ parentfolder.close();
+ parentfolder = null;
+ } catch (Exception ex) {
+ // Ignore
+ }
+ }
+ }
+
+ return resultat;
+ }
+
+ /**
+ * Make sure that we are not loading jobs recursively...
+ *
+ * @param parentJob the parent job
+ * @param jobMeta the job metadata
+ * @throws KettleException in case both jobs are loaded from the same source
+ */
+ private void verifyRecursiveExecution(Job parentJob, JobMeta jobMeta) throws KettleException {
+
+ if (parentJob == null) {
+ return; // OK!
+ }
+
+ JobMeta parentJobMeta = parentJob.getJobMeta();
+
+ if (parentJobMeta.getName() == null && jobMeta.getName() != null) {
+ return; // OK
+ }
+ if (parentJobMeta.getName() != null && jobMeta.getName() == null) {
+ return; // OK as well.
+ }
+
+ // Not from the repository? just verify the filename
+ //
+ if (jobMeta.getFilename() != null && jobMeta.getFilename().equals(parentJobMeta.getFilename())) {
+ throw new KettleException(BaseMessages.getString(PKG, "JobJobError.Recursive", jobMeta.getFilename()));
+ }
+
+ // Different directories: OK
+ if (parentJobMeta.getRepositoryDirectory() == null && jobMeta.getRepositoryDirectory() != null) {
+ return;
+ }
+ if (parentJobMeta.getRepositoryDirectory() != null && jobMeta.getRepositoryDirectory() == null) {
+ return;
+ }
+ if (jobMeta.getRepositoryDirectory().getObjectId() != parentJobMeta.getRepositoryDirectory().getObjectId()) {
+ return;
+ }
+
+ // Same names, same directories : loaded from same location in the
+ // repository:
+ // --> recursive loading taking place!
+ //
+ if (parentJobMeta.getName().equals(jobMeta.getName())) {
+ throw new KettleException(BaseMessages.getString(PKG, "JobJobError.Recursive", jobMeta.getFilename()));
+ }
+
+ // Also compare with the grand-parent (if there is any)
+ verifyRecursiveExecution(parentJob.getParentJob(), jobMeta);
+ }
+
+ public void clear() {
+ super.clear();
+
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+ jobname = null;
+ filename = null;
+ directory = null;
+ arguments = null;
+ argFromPrevious = false;
+ addDate = false;
+ addTime = false;
+ logfile = null;
+ logext = null;
+ setLogfile = false;
+ setAppendLogfile = false;
+ }
+
+ public boolean evaluates() {
+ return true;
+ }
+
+ public boolean isUnconditional() {
+ return true;
+ }
+
+ public List getSQLStatements(Repository repository, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ this.copyVariablesFrom(space);
+ JobMeta jobMeta = getJobMeta(repository, metaStore, space);
+ return jobMeta.getSQLStatements(repository, null);
+ }
+
+ @Deprecated
+ public JobMeta getJobMeta(Repository rep, VariableSpace space) throws KettleException {
+ if (rep != null) {
+ return getJobMeta(rep, rep.getMetaStore(), space);
+ } else {
+ return getJobMeta(rep, getMetaStore(), space);
+ }
+ }
+
+ public JobMeta getJobMeta(Repository rep, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ JobMeta jobMeta = null;
+ try {
+ CurrentDirectoryResolver r = new CurrentDirectoryResolver();
+ VariableSpace tmpSpace = r.resolveCurrentDirectory(
+ specificationMethod, space, rep, parentJob, getFilename());
+ switch (specificationMethod) {
+ case FILENAME:
+ String realFilename = tmpSpace.environmentSubstitute(getFilename());
+ if (rep != null) {
+ // need to try to load from the repository
+ realFilename = r.normalizeSlashes(realFilename);
+ try {
+ String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
+ String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1);
+ RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
+ // jobMeta = rep.loadJob(tmpFilename, dir, null, null);
+ jobMeta = ResourceDefinitionHelper.loadJob(rep, dir, tmpFilename);
+ } catch (KettleException ke) {
+ // try without extension
+ if (realFilename.endsWith(Const.STRING_JOB_DEFAULT_EXT)) {
+ try {
+ String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1,
+ realFilename.indexOf("." + Const.STRING_JOB_DEFAULT_EXT));
+ String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
+ RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
+ jobMeta = rep.loadJob(tmpFilename, dir, null, null);
+ } catch (KettleException ke2) {
+ // fall back to try loading from file system (mappingJobMeta is going to be null)
+ }
+ }
+ }
+ }
+ if (jobMeta == null) {
+ jobMeta = new JobMeta(tmpSpace, realFilename, rep, metaStore, null);
+ }
+ break;
+ case REPOSITORY_BY_NAME:
+ String realDirectory = tmpSpace.environmentSubstitute(getDirectory());
+ String realJobName = tmpSpace.environmentSubstitute(getJobName());
+
+ if (rep != null) {
+ realDirectory = r.normalizeSlashes(realDirectory);
+ RepositoryDirectoryInterface repositoryDirectory =
+ rep.loadRepositoryDirectoryTree().findDirectory(realDirectory);
+ if (repositoryDirectory == null) {
+ throw new KettleException("Unable to find repository directory ["
+ + Const.NVL(realDirectory, "") + "]");
+ }
+ jobMeta = rep.loadJob(realJobName, repositoryDirectory, null, null); // reads
+ } else {
+ // rep is null, let's try loading by filename
+ try {
+ jobMeta = new JobMeta(tmpSpace, realDirectory + "/" + realJobName, rep, metaStore, null);
+ } catch (KettleException ke) {
+ try {
+ // add .kjb extension and try again
+ jobMeta = new JobMeta(tmpSpace,
+ realDirectory + "/" + realJobName + "." + Const.STRING_JOB_DEFAULT_EXT, rep, metaStore, null);
+ } catch (KettleException ke2) {
+ ke2.printStackTrace();
+ throw new KettleException(
+ "Could not execute job specified in a repository since we're not connected to one");
+ }
+ }
+ }
+ break;
+ case REPOSITORY_BY_REFERENCE:
+ if (rep != null) {
+ // Load the last version...
+ //
+ jobMeta = rep.loadJob(jobObjectId, null);
+ break;
+ } else {
+ throw new KettleException(
+ "Could not execute job specified in a repository since we're not connected to one");
+ }
+ default:
+ throw new KettleException("The specified object location specification method '"
+ + specificationMethod + "' is not yet supported in this job entry.");
+ }
+
+ if (jobMeta != null) {
+ jobMeta.setRepository(rep);
+ jobMeta.setMetaStore(metaStore);
+ }
+
+ return jobMeta;
+ } catch (Exception e) {
+ throw new KettleException("Unexpected error during job metadata load", e);
+ }
+
+ }
+
+ /**
+ * @return Returns the runEveryResultRow.
+ */
+ public boolean isExecPerRow() {
+ return execPerRow;
+ }
+
+ /**
+ * @param runEveryResultRow The runEveryResultRow to set.
+ */
+ public void setExecPerRow(boolean runEveryResultRow) {
+ this.execPerRow = runEveryResultRow;
+ }
+
+ public List getResourceDependencies(JobMeta jobMeta) {
+ List references = super.getResourceDependencies(jobMeta);
+ if (!Const.isEmpty(filename)) {
+ String realFileName = jobMeta.environmentSubstitute(filename);
+ ResourceReference reference = new ResourceReference(this);
+ reference.getEntries().add(new ResourceEntry(realFileName, ResourceType.ACTIONFILE));
+ references.add(reference);
+ }
+ return references;
+ }
+
+ /**
+ * Exports the object to a flat-file system, adding content with filename keys to a set of definitions. The supplied
+ * resource naming interface allows the object to name appropriately without worrying about those parts of the
+ * implementation specific details.
+ *
+ * @param space The variable space to resolve (environment) variables with.
+ * @param definitions The map containing the filenames and content
+ * @param namingInterface The resource naming interface allows the object to be named appropriately
+ * @param repository The repository to load resources from
+ * @param metaStore the metaStore to load external metadata from
+ * @return The filename for this object. (also contained in the definitions map)
+ * @throws KettleException in case something goes wrong during the export
+ */
+ public String exportResources(VariableSpace space, Map definitions,
+ ResourceNamingInterface namingInterface, Repository repository, IMetaStore metaStore) throws KettleException {
+ // Try to load the transformation from repository or file.
+ // Modify this recursively too...
+ //
+ // AGAIN: there is no need to clone this job entry because the caller is
+ // responsible for this.
+ //
+ // First load the job meta data...
+ //
+ copyVariablesFrom(space); // To make sure variables are available.
+ JobMeta jobMeta = getJobMeta(repository, metaStore, space);
+
+ if (jobMeta instanceof ResourceDefinitionHelper.JobMetaCollection) {
+ ResourceDefinitionHelper.JobMetaCollection jmc = (ResourceDefinitionHelper.JobMetaCollection) jobMeta;
+ for (JobMeta j : jmc.getAttachedMeta()) {
+ if (!ResourceDefinitionHelper.containsResource(definitions, space, namingInterface, j)) {
+ j.exportResources(jobMeta, definitions, namingInterface, repository, metaStore);
+ }
+ }
+
+ return null;
+ }
+
+ // Also go down into the job and export the files there. (going down
+ // recursively)
+ //
+ String proposedNewFilename =
+ jobMeta.exportResources(jobMeta, definitions, namingInterface, repository, metaStore);
+
+ // To get a relative path to it, we inject
+ // ${Internal.Job.Filename.Directory}
+ //
+ String newFilename = "${" + Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY + "}/" + proposedNewFilename;
+
+ // Set the filename in the job
+ //
+ jobMeta.setFilename(newFilename);
+
+ // exports always reside in the root directory, in case we want to turn this
+ // into a file repository...
+ //
+ jobMeta.setRepositoryDirectory(new RepositoryDirectory());
+
+ // export to filename ALWAYS (this allows the exported XML to be executed remotely)
+ //
+ setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
+
+ // change it in the job entry
+ //
+ filename = newFilename;
+
+ return proposedNewFilename;
+ }
+
+ @Override
+ public void check(List remarks, JobMeta jobMeta, VariableSpace space,
+ Repository repository, IMetaStore metaStore) {
+ if (setLogfile) {
+ JobEntryValidatorUtils.andValidator().validate(this, "logfile", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ }
+
+ if (null != directory) {
+ // if from repo
+ JobEntryValidatorUtils.andValidator().validate(this, "directory", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notNullValidator()));
+ JobEntryValidatorUtils.andValidator().validate(this, "jobName", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ } else {
+ // else from xml file
+ JobEntryValidatorUtils.andValidator().validate(this, "filename", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ }
+ }
+
+ public static void main(String[] args) {
+ List remarks = new ArrayList();
+ new JobEntryJob().check(remarks, null, new Variables(), null, null);
+ System.out.printf("Remarks: %s\n", remarks);
+ }
+
+ protected String getLogfile() {
+ return logfile;
+ }
+
+ /**
+ * @return the remote slave server name
+ */
+ public String getRemoteSlaveServerName() {
+ return remoteSlaveServerName;
+ }
+
+ /**
+ * @param remoteSlaveServerName the remoteSlaveServer to set
+ */
+ public void setRemoteSlaveServerName(String remoteSlaveServerName) {
+ this.remoteSlaveServerName = remoteSlaveServerName;
+ }
+
+ /**
+ * @return the waitingToFinish
+ */
+ public boolean isWaitingToFinish() {
+ return waitingToFinish;
+ }
+
+ /**
+ * @param waitingToFinish the waitingToFinish to set
+ */
+ public void setWaitingToFinish(boolean waitingToFinish) {
+ this.waitingToFinish = waitingToFinish;
+ }
+
+ /**
+ * @return the followingAbortRemotely
+ */
+ public boolean isFollowingAbortRemotely() {
+ return followingAbortRemotely;
+ }
+
+ /**
+ * @param followingAbortRemotely the followingAbortRemotely to set
+ */
+ public void setFollowingAbortRemotely(boolean followingAbortRemotely) {
+ this.followingAbortRemotely = followingAbortRemotely;
+ }
+
+ /**
+ * @return the passingAllParameters
+ */
+ public boolean isPassingAllParameters() {
+ return passingAllParameters;
+ }
+
+ /**
+ * @param passingAllParameters the passingAllParameters to set
+ */
+ public void setPassingAllParameters(boolean passingAllParameters) {
+ this.passingAllParameters = passingAllParameters;
+ }
+
+ public Job getJob() {
+ return job;
+ }
+
+ /**
+ * @return the jobObjectId
+ */
+ public ObjectId getJobObjectId() {
+ return jobObjectId;
+ }
+
+ /**
+ * @param jobObjectId the jobObjectId to set
+ */
+ public void setJobObjectId(ObjectId jobObjectId) {
+ this.jobObjectId = jobObjectId;
+ }
+
+ /**
+ * @return the specificationMethod
+ */
+ public ObjectLocationSpecificationMethod getSpecificationMethod() {
+ return specificationMethod;
+ }
+
+ /**
+ * @param specificationMethod the specificationMethod to set
+ */
+ public void setSpecificationMethod(ObjectLocationSpecificationMethod specificationMethod) {
+ this.specificationMethod = specificationMethod;
+ }
+
+ @Override
+ public boolean hasRepositoryReferences() {
+ return specificationMethod == ObjectLocationSpecificationMethod.REPOSITORY_BY_REFERENCE;
+ }
+
+ /**
+ * Look up the references after import
+ *
+ * @param repository the repository to reference.
+ */
+ public void lookupRepositoryReferences(Repository repository) throws KettleException {
+ // The correct reference is stored in the job name and directory attributes...
+ //
+ RepositoryDirectoryInterface repositoryDirectoryInterface =
+ RepositoryImportLocation.getRepositoryImportLocation().findDirectory(directory);
+ jobObjectId = repository.getJobId(jobname, repositoryDirectoryInterface);
+ }
+
+ private boolean isJobDefined() {
+ return !Const.isEmpty(filename)
+ || jobObjectId != null || (!Const.isEmpty(this.directory) && !Const.isEmpty(jobname));
+ }
+
+ public boolean[] isReferencedObjectEnabled() {
+ return new boolean[]{isJobDefined(),};
+ }
+
+ /**
+ * @return The objects referenced in the step, like a a transformation, a job, a mapper, a reducer, a combiner, ...
+ */
+ public String[] getReferencedObjectDescriptions() {
+ return new String[]{BaseMessages.getString(PKG, "JobEntryJob.ReferencedObject.Description"),};
+ }
+
+ /**
+ * Load the referenced object
+ *
+ * @param index the referenced object index to load (in case there are multiple references)
+ * @param rep the repository
+ * @param metaStore the metaStore
+ * @param space the variable space to use
+ * @return the referenced object once loaded
+ * @throws KettleException
+ */
+ public Object loadReferencedObject(int index, Repository rep, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ return getJobMeta(rep, metaStore, space);
+ }
+
+ public boolean isExpandingRemoteJob() {
+ return expandingRemoteJob;
+ }
+
+ public void setExpandingRemoteJob(boolean expandingRemoteJob) {
+ this.expandingRemoteJob = expandingRemoteJob;
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/trans/JobEntryTrans.java b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/trans/JobEntryTrans.java
new file mode 100644
index 0000000..8072c06
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/job/entries/trans/JobEntryTrans.java
@@ -0,0 +1,1567 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.job.entries.trans;
+
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.*;
+import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.exception.KettleDatabaseException;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.exception.KettleXMLException;
+import org.pentaho.di.core.logging.LogChannelFileWriter;
+import org.pentaho.di.core.logging.LogLevel;
+import org.pentaho.di.core.parameters.NamedParams;
+import org.pentaho.di.core.parameters.NamedParamsDefault;
+import org.pentaho.di.core.util.CurrentDirectoryResolver;
+import org.pentaho.di.core.util.FileUtil;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.core.xml.XMLHandler;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.DelegationListener;
+import org.pentaho.di.job.Job;
+import org.pentaho.di.job.JobMeta;
+import org.pentaho.di.job.entries.RemoteJobEntryLogHelper;
+import org.pentaho.di.job.entry.JobEntryBase;
+import org.pentaho.di.job.entry.JobEntryInterface;
+import org.pentaho.di.job.entry.validator.AndValidator;
+import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;
+import org.pentaho.di.repository.*;
+import org.pentaho.di.resource.*;
+import org.pentaho.di.resource.ResourceEntry.ResourceType;
+import org.pentaho.di.trans.Trans;
+import org.pentaho.di.trans.TransExecutionConfiguration;
+import org.pentaho.di.trans.TransMeta;
+import org.pentaho.di.trans.cluster.TransSplitter;
+import org.pentaho.di.trans.step.StepMeta;
+import org.pentaho.di.www.SlaveServerTransStatus;
+import org.pentaho.metastore.api.IMetaStore;
+import org.w3c.dom.Node;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This is the job entry that defines a transformation to be run.
+ *
+ * @author Matt Casters
+ * @since 1-Oct-2003, rewritten on 18-June-2004
+ */
+public class JobEntryTrans extends JobEntryBase implements Cloneable, JobEntryInterface {
+ private static Class PKG = JobEntryTrans.class; // for i18n purposes, needed by Translator2!!
+
+ private String transname;
+
+ private String filename;
+
+ private String directory;
+
+ private ObjectId transObjectId;
+
+ private ObjectLocationSpecificationMethod specificationMethod;
+
+ public String[] arguments;
+
+ public boolean argFromPrevious;
+
+ public boolean paramsFromPrevious;
+
+ public boolean execPerRow;
+
+ public String[] parameters;
+
+ public String[] parameterFieldNames;
+
+ public String[] parameterValues;
+
+ public boolean clearResultRows;
+
+ public boolean clearResultFiles;
+
+ public boolean createParentFolder;
+
+ public boolean setLogfile;
+
+ public boolean setAppendLogfile;
+
+ public String logfile, logext;
+
+ public boolean addDate, addTime;
+
+ public LogLevel logFileLevel;
+
+ private String directoryPath;
+
+ private boolean clustering;
+
+ public boolean waitingToFinish = true;
+
+ public boolean followingAbortRemotely;
+
+ private String remoteSlaveServerName;
+
+ private boolean passingAllParameters = true;
+
+ private boolean loggingRemoteWork;
+
+ private Trans trans;
+
+ public JobEntryTrans(String name) {
+ super(name, "");
+ }
+
+ public JobEntryTrans() {
+ this("");
+ clear();
+ }
+
+ private void allocateArgs(int nrArgs) {
+ arguments = new String[nrArgs];
+ }
+
+ private void allocateParams(int nrParameters) {
+ parameters = new String[nrParameters];
+ parameterFieldNames = new String[nrParameters];
+ parameterValues = new String[nrParameters];
+ }
+
+ public Object clone() {
+ JobEntryTrans je = (JobEntryTrans) super.clone();
+ if (arguments != null) {
+ int nrArgs = arguments.length;
+ je.allocateArgs(nrArgs);
+ System.arraycopy(arguments, 0, je.arguments, 0, nrArgs);
+ }
+ if (parameters != null) {
+ int nrParameters = parameters.length;
+ je.allocateParams(nrParameters);
+ System.arraycopy(parameters, 0, je.parameters, 0, nrParameters);
+ System.arraycopy(parameterFieldNames, 0, je.parameterFieldNames, 0, nrParameters);
+ System.arraycopy(parameterValues, 0, je.parameterValues, 0, nrParameters);
+ }
+ return je;
+ }
+
+ public void setFileName(String n) {
+ filename = n;
+ }
+
+ /**
+ * @return the filename
+ * @deprecated use getFilename() instead
+ */
+ @Deprecated
+ public String getFileName() {
+ return filename;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+
+ public String getRealFilename() {
+ return environmentSubstitute(getFilename());
+ }
+
+ public void setTransname(String transname) {
+ this.transname = transname;
+ }
+
+ public String getTransname() {
+ return transname;
+ }
+
+ public String getDirectory() {
+ return directory;
+ }
+
+ public void setDirectory(String directory) {
+ this.directory = directory;
+ }
+
+ public String getLogFilename() {
+ String retval = "";
+ if (setLogfile) {
+ retval += logfile == null ? "" : logfile;
+ Calendar cal = Calendar.getInstance();
+ if (addDate) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+ retval += "_" + sdf.format(cal.getTime());
+ }
+ if (addTime) {
+ SimpleDateFormat sdf = new SimpleDateFormat("HHmmss");
+ retval += "_" + sdf.format(cal.getTime());
+ }
+ if (logext != null && logext.length() > 0) {
+ retval += "." + logext;
+ }
+ }
+ return retval;
+ }
+
+ public String getXML() {
+ StringBuffer retval = new StringBuffer(300);
+
+ retval.append(super.getXML());
+
+ // specificationMethod
+ //
+ retval.append(" ").append(
+ XMLHandler.addTagValue("specification_method", specificationMethod == null ? null : specificationMethod
+ .getCode())
+ );
+ retval.append(" ").append(
+ XMLHandler.addTagValue("trans_object_id", transObjectId == null ? null : transObjectId.toString()));
+ // Export a little bit of extra information regarding the reference since it doesn't really matter outside the same
+ // repository.
+ //
+ if (rep != null && transObjectId != null) {
+ try {
+ RepositoryObject objectInformation =
+ rep.getObjectInformation(transObjectId, RepositoryObjectType.TRANSFORMATION);
+ if (objectInformation != null) {
+ transname = objectInformation.getName();
+ directory = objectInformation.getRepositoryDirectory().getPath();
+ }
+ } catch (KettleException e) {
+ // Ignore object reference problems. It simply means that the reference is no longer valid.
+ }
+ }
+ retval.append(" ").append(XMLHandler.addTagValue("filename", filename));
+ retval.append(" ").append(XMLHandler.addTagValue("transname", transname));
+
+ if (directory != null) {
+ retval.append(" ").append(XMLHandler.addTagValue("directory", directory));
+ } else if (directoryPath != null) {
+ // don't loose this info (backup/recovery)
+ //
+ retval.append(" ").append(XMLHandler.addTagValue("directory", directoryPath));
+ }
+ retval.append(" ").append(XMLHandler.addTagValue("arg_from_previous", argFromPrevious));
+ retval.append(" ").append(XMLHandler.addTagValue("params_from_previous", paramsFromPrevious));
+ retval.append(" ").append(XMLHandler.addTagValue("exec_per_row", execPerRow));
+ retval.append(" ").append(XMLHandler.addTagValue("clear_rows", clearResultRows));
+ retval.append(" ").append(XMLHandler.addTagValue("clear_files", clearResultFiles));
+ retval.append(" ").append(XMLHandler.addTagValue("set_logfile", setLogfile));
+ retval.append(" ").append(XMLHandler.addTagValue("logfile", logfile));
+ retval.append(" ").append(XMLHandler.addTagValue("logext", logext));
+ retval.append(" ").append(XMLHandler.addTagValue("add_date", addDate));
+ retval.append(" ").append(XMLHandler.addTagValue("add_time", addTime));
+ retval.append(" ").append(
+ XMLHandler.addTagValue("loglevel", logFileLevel != null ? logFileLevel.getCode() : null));
+ retval.append(" ").append(XMLHandler.addTagValue("cluster", clustering));
+ retval.append(" ").append(XMLHandler.addTagValue("slave_server_name", remoteSlaveServerName));
+ retval.append(" ").append(XMLHandler.addTagValue("set_append_logfile", setAppendLogfile));
+ retval.append(" ").append(XMLHandler.addTagValue("wait_until_finished", waitingToFinish));
+ retval.append(" ").append(XMLHandler.addTagValue("follow_abort_remote", followingAbortRemotely));
+ retval.append(" ").append(XMLHandler.addTagValue("create_parent_folder", createParentFolder));
+ retval.append(" ").append(XMLHandler.addTagValue("logging_remote_work", loggingRemoteWork));
+
+ if (arguments != null) {
+ for (int i = 0; i < arguments.length; i++) {
+ // This is a very very bad way of making an XML file, don't use it (or
+ // copy it). Sven Boden
+ retval.append(" ").append(XMLHandler.addTagValue("argument" + i, arguments[i]));
+ }
+ }
+
+ if (parameters != null) {
+ retval.append(" ").append(XMLHandler.openTag("parameters"));
+
+ retval.append(" ").append(XMLHandler.addTagValue("pass_all_parameters", passingAllParameters));
+
+ for (int i = 0; i < parameters.length; i++) {
+ // This is a better way of making the XML file than the arguments.
+ retval.append(" ").append(XMLHandler.openTag("parameter"));
+
+ retval.append(" ").append(XMLHandler.addTagValue("name", parameters[i]));
+ retval.append(" ").append(XMLHandler.addTagValue("stream_name", parameterFieldNames[i]));
+ retval.append(" ").append(XMLHandler.addTagValue("value", parameterValues[i]));
+
+ retval.append(" ").append(XMLHandler.closeTag("parameter"));
+ }
+ retval.append(" ").append(XMLHandler.closeTag("parameters"));
+ }
+
+ return retval.toString();
+ }
+
+ private void checkObjectLocationSpecificationMethod() {
+ if (specificationMethod == null) {
+ // Backward compatibility
+ //
+ // Default = Filename
+ //
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+
+ if (!Const.isEmpty(filename)) {
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+ } else if (transObjectId != null) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_REFERENCE;
+ } else if (!Const.isEmpty(transname)) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_NAME;
+ }
+ }
+ }
+
+ public void loadXML(Node entrynode, List databases, List slaveServers,
+ Repository rep, IMetaStore metaStore) throws KettleXMLException {
+ try {
+ super.loadXML(entrynode, databases, slaveServers);
+
+ String method = XMLHandler.getTagValue(entrynode, "specification_method");
+ specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
+
+ String transId = XMLHandler.getTagValue(entrynode, "trans_object_id");
+ transObjectId = Const.isEmpty(transId) ? null : new StringObjectId(transId);
+ filename = XMLHandler.getTagValue(entrynode, "filename");
+ transname = XMLHandler.getTagValue(entrynode, "transname");
+ directory = XMLHandler.getTagValue(entrynode, "directory");
+
+ if (rep != null && rep.isConnected() && !Const.isEmpty(transname)) {
+ specificationMethod = ObjectLocationSpecificationMethod.REPOSITORY_BY_NAME;
+ }
+
+ // Backward compatibility check for object specification
+ //
+ checkObjectLocationSpecificationMethod();
+
+ argFromPrevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "arg_from_previous"));
+ paramsFromPrevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "params_from_previous"));
+ execPerRow = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "exec_per_row"));
+ clearResultRows = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "clear_rows"));
+ clearResultFiles = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "clear_files"));
+ setLogfile = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "set_logfile"));
+ addDate = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "add_date"));
+ addTime = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "add_time"));
+ logfile = XMLHandler.getTagValue(entrynode, "logfile");
+ logext = XMLHandler.getTagValue(entrynode, "logext");
+ logFileLevel = LogLevel.getLogLevelForCode(XMLHandler.getTagValue(entrynode, "loglevel"));
+ clustering = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "cluster"));
+ createParentFolder = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "create_parent_folder"));
+ loggingRemoteWork = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "logging_remote_work"));
+
+ remoteSlaveServerName = XMLHandler.getTagValue(entrynode, "slave_server_name");
+
+ setAppendLogfile = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "set_append_logfile"));
+ String wait = XMLHandler.getTagValue(entrynode, "wait_until_finished");
+ if (Const.isEmpty(wait)) {
+ waitingToFinish = true;
+ } else {
+ waitingToFinish = "Y".equalsIgnoreCase(wait);
+ }
+
+ followingAbortRemotely = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "follow_abort_remote"));
+
+ // How many arguments?
+ int argnr = 0;
+ while (XMLHandler.getTagValue(entrynode, "argument" + argnr) != null) {
+ argnr++;
+ }
+ allocateArgs(argnr);
+
+ // Read them all...
+ for (int a = 0; a < argnr; a++) {
+ arguments[a] = XMLHandler.getTagValue(entrynode, "argument" + a);
+ }
+
+ Node parametersNode = XMLHandler.getSubNode(entrynode, "parameters");
+
+ String passAll = XMLHandler.getTagValue(parametersNode, "pass_all_parameters");
+ passingAllParameters = Const.isEmpty(passAll) || "Y".equalsIgnoreCase(passAll);
+
+ int nrParameters = XMLHandler.countNodes(parametersNode, "parameter");
+ allocateParams(nrParameters);
+
+ for (int i = 0; i < nrParameters; i++) {
+ Node knode = XMLHandler.getSubNodeByNr(parametersNode, "parameter", i);
+
+ parameters[i] = XMLHandler.getTagValue(knode, "name");
+ parameterFieldNames[i] = XMLHandler.getTagValue(knode, "stream_name");
+ parameterValues[i] = XMLHandler.getTagValue(knode, "value");
+ }
+ } catch (KettleException e) {
+ throw new KettleXMLException("Unable to load job entry of type 'trans' from XML node", e);
+ }
+ }
+
+ // Load the jobentry from repository
+ //
+ public void loadRep(Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List databases,
+ List slaveServers) throws KettleException {
+ try {
+ String method = rep.getJobEntryAttributeString(id_jobentry, "specification_method");
+ specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
+ String transId = rep.getJobEntryAttributeString(id_jobentry, "trans_object_id");
+ transObjectId = Const.isEmpty(transId) ? null : new StringObjectId(transId);
+ transname = rep.getJobEntryAttributeString(id_jobentry, "name");
+ directory = rep.getJobEntryAttributeString(id_jobentry, "dir_path");
+ filename = rep.getJobEntryAttributeString(id_jobentry, "file_name");
+
+ // Backward compatibility check for object specification
+ //
+ checkObjectLocationSpecificationMethod();
+
+ argFromPrevious = rep.getJobEntryAttributeBoolean(id_jobentry, "arg_from_previous");
+ paramsFromPrevious = rep.getJobEntryAttributeBoolean(id_jobentry, "params_from_previous");
+ execPerRow = rep.getJobEntryAttributeBoolean(id_jobentry, "exec_per_row");
+ clearResultRows = rep.getJobEntryAttributeBoolean(id_jobentry, "clear_rows", true);
+ clearResultFiles = rep.getJobEntryAttributeBoolean(id_jobentry, "clear_files", true);
+ setLogfile = rep.getJobEntryAttributeBoolean(id_jobentry, "set_logfile");
+ addDate = rep.getJobEntryAttributeBoolean(id_jobentry, "add_date");
+ addTime = rep.getJobEntryAttributeBoolean(id_jobentry, "add_time");
+ logfile = rep.getJobEntryAttributeString(id_jobentry, "logfile");
+ logext = rep.getJobEntryAttributeString(id_jobentry, "logext");
+ logFileLevel = LogLevel.getLogLevelForCode(rep.getJobEntryAttributeString(id_jobentry, "loglevel"));
+ clustering = rep.getJobEntryAttributeBoolean(id_jobentry, "cluster");
+ createParentFolder = rep.getJobEntryAttributeBoolean(id_jobentry, "create_parent_folder");
+
+ remoteSlaveServerName = rep.getJobEntryAttributeString(id_jobentry, "slave_server_name");
+ setAppendLogfile = rep.getJobEntryAttributeBoolean(id_jobentry, "set_append_logfile");
+ waitingToFinish = rep.getJobEntryAttributeBoolean(id_jobentry, "wait_until_finished", true);
+ followingAbortRemotely = rep.getJobEntryAttributeBoolean(id_jobentry, "follow_abort_remote");
+ loggingRemoteWork = rep.getJobEntryAttributeBoolean(id_jobentry, "logging_remote_work");
+
+ // How many arguments?
+ int argnr = rep.countNrJobEntryAttributes(id_jobentry, "argument");
+ allocateArgs(argnr);
+
+ // Read all arguments...
+ for (int a = 0; a < argnr; a++) {
+ arguments[a] = rep.getJobEntryAttributeString(id_jobentry, a, "argument");
+ }
+
+ // How many arguments?
+ int parameternr = rep.countNrJobEntryAttributes(id_jobentry, "parameter_name");
+ allocateParams(parameternr);
+
+ // Read all parameters ...
+ for (int a = 0; a < parameternr; a++) {
+ parameters[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_name");
+ parameterFieldNames[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_stream_name");
+ parameterValues[a] = rep.getJobEntryAttributeString(id_jobentry, a, "parameter_value");
+ }
+
+ passingAllParameters = rep.getJobEntryAttributeBoolean(id_jobentry, "pass_all_parameters", true);
+
+ } catch (KettleDatabaseException dbe) {
+ throw new KettleException("Unable to load job entry of type 'trans' from the repository for id_jobentry="
+ + id_jobentry, dbe);
+ }
+ }
+
+ // Save the attributes of this job entry
+ //
+ public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_job) throws KettleException {
+ try {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "specification_method", specificationMethod == null
+ ? null : specificationMethod.getCode());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "trans_object_id", transObjectId == null
+ ? null : transObjectId.toString());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "name", getTransname());
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "dir_path", getDirectory() != null ? getDirectory() : "");
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "file_name", filename);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "arg_from_previous", argFromPrevious);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "params_from_previous", paramsFromPrevious);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "exec_per_row", execPerRow);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "clear_rows", clearResultRows);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "clear_files", clearResultFiles);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "set_logfile", setLogfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "add_date", addDate);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "add_time", addTime);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "logfile", logfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "logext", logext);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "loglevel", logFileLevel != null
+ ? logFileLevel.getCode() : null);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "cluster", clustering);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "slave_server_name", remoteSlaveServerName);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "set_append_logfile", setAppendLogfile);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "wait_until_finished", waitingToFinish);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "follow_abort_remote", followingAbortRemotely);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "create_parent_folder", createParentFolder);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "logging_remote_work", loggingRemoteWork);
+
+ // Save the arguments...
+ if (arguments != null) {
+ for (int i = 0; i < arguments.length; i++) {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "argument", arguments[i]);
+ }
+ }
+
+ // Save the parameters...
+ if (parameters != null) {
+ for (int i = 0; i < parameters.length; i++) {
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_name", parameters[i]);
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_stream_name", Const.NVL(
+ parameterFieldNames[i], ""));
+ rep.saveJobEntryAttribute(id_job, getObjectId(), i, "parameter_value", Const.NVL(
+ parameterValues[i], ""));
+ }
+ }
+
+ rep.saveJobEntryAttribute(id_job, getObjectId(), "pass_all_parameters", passingAllParameters);
+
+ } catch (KettleDatabaseException dbe) {
+ throw new KettleException(
+ "Unable to save job entry of type 'trans' to the repository for id_job=" + id_job, dbe);
+ }
+ }
+
+ public void clear() {
+ super.clear();
+
+ specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
+ transname = null;
+ filename = null;
+ directory = null;
+ arguments = null;
+ argFromPrevious = false;
+ execPerRow = false;
+ addDate = false;
+ addTime = false;
+ logfile = null;
+ logext = null;
+ setLogfile = false;
+ clearResultRows = false;
+ clearResultFiles = false;
+ remoteSlaveServerName = null;
+ setAppendLogfile = false;
+ waitingToFinish = true;
+ followingAbortRemotely = false; // backward compatibility reasons
+ createParentFolder = false;
+ logFileLevel = LogLevel.BASIC;
+ }
+
+ /**
+ * Execute this job entry and return the result. In this case it means, just set the result boolean in the Result
+ * class.
+ *
+ * @param result The result of the previous execution
+ * @param nr the job entry number
+ * @return The Result of the execution.
+ */
+ public Result execute(Result result, int nr) throws KettleException {
+ result.setEntryNr(nr);
+
+ LogChannelFileWriter logChannelFileWriter = null;
+
+ LogLevel transLogLevel = parentJob.getLogLevel();
+
+ String realLogFilename = "";
+ if (setLogfile) {
+ transLogLevel = logFileLevel;
+
+ realLogFilename = environmentSubstitute(getLogFilename());
+
+ // We need to check here the log filename
+ // if we do not have one, we must fail
+ if (Const.isEmpty(realLogFilename)) {
+ logError(BaseMessages.getString(PKG, "JobTrans.Exception.LogFilenameMissing"));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ // create parent folder?
+ if (!FileUtil.createParentFolder(PKG, realLogFilename, createParentFolder, this.getLogChannel(), this)) {
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ try {
+ logChannelFileWriter =
+ new LogChannelFileWriter(
+ this.getLogChannelId(), KettleVFS.getFileObject(realLogFilename), setAppendLogfile);
+ logChannelFileWriter.startLogging();
+ } catch (KettleException e) {
+ logError(BaseMessages.getString(PKG, "JobTrans.Error.UnableOpenAppender", realLogFilename, e.toString()));
+
+ logError(Const.getStackTracker(e));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ }
+
+ // Figure out the remote slave server...
+ //
+ SlaveServer remoteSlaveServer = null;
+ if (!Const.isEmpty(remoteSlaveServerName)) {
+ String realRemoteSlaveServerName = environmentSubstitute(remoteSlaveServerName);
+ remoteSlaveServer = parentJob.getJobMeta().findSlaveServer(realRemoteSlaveServerName);
+ if (remoteSlaveServer == null) {
+ throw new KettleException(BaseMessages.getString(
+ PKG, "JobTrans.Exception.UnableToFindRemoteSlaveServer", realRemoteSlaveServerName));
+ }
+ }
+
+ // Open the transformation...
+ //
+ switch (specificationMethod) {
+ case FILENAME:
+ if (isDetailed()) {
+ logDetailed(BaseMessages.getString(
+ PKG, "JobTrans.Log.OpeningTrans", environmentSubstitute(getFilename())));
+ }
+ break;
+ case REPOSITORY_BY_NAME:
+ if (isDetailed()) {
+ logDetailed(BaseMessages.getString(
+ PKG, "JobTrans.Log.OpeningTransInDirec", environmentSubstitute(getFilename()),
+ environmentSubstitute(directory)));
+ }
+ break;
+ case REPOSITORY_BY_REFERENCE:
+ if (isDetailed()) {
+ logDetailed(BaseMessages.getString(PKG, "JobTrans.Log.OpeningTransByReference", transObjectId));
+ }
+ break;
+ default:
+ break;
+ }
+
+ // Load the transformation only once for the complete loop!
+ // Throws an exception if it was not possible to load the transformation. For example, the XML file doesn't exist or
+ // the repository is down.
+ // Log the stack trace and return an error condition from this
+ //
+ TransMeta transMeta = null;
+ try {
+ transMeta = getTransMeta(rep, metaStore, this);
+ } catch (KettleException e) {
+ logError(Const.getStackTracker(e));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+
+ int iteration = 0;
+ String[] args1 = arguments;
+ if (args1 == null || args1.length == 0) { // No arguments set, look at the parent job.
+ args1 = parentJob.getArguments();
+ }
+ // initializeVariablesFrom(parentJob);
+
+ //
+ // For the moment only do variable translation at the start of a job, not
+ // for every input row (if that would be switched on). This is for safety,
+ // the real argument setting is later on.
+ //
+ String[] args = null;
+ if (args1 != null) {
+ args = new String[args1.length];
+ for (int idx = 0; idx < args1.length; idx++) {
+ args[idx] = environmentSubstitute(args1[idx]);
+ }
+ }
+
+ RowMetaAndData resultRow = null;
+ boolean first = true;
+ List rows = new ArrayList(result.getRows());
+
+ while ((first && !execPerRow)
+ || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)
+ && !parentJob.isStopped()) {
+ // Clear the result rows of the result
+ // Otherwise we double the amount of rows every iteration in the simple cases.
+ //
+ if (execPerRow) {
+ result.getRows().clear();
+ }
+ if (rows != null && execPerRow) {
+ resultRow = rows.get(iteration);
+ } else {
+ resultRow = null;
+ }
+
+ NamedParams namedParam = new NamedParamsDefault();
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+ // We have a parameter
+ //
+ namedParam.addParameterDefinition(parameters[idx], "", "Job entry runtime");
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ // There is no field name specified.
+ //
+ String value = Const.NVL(environmentSubstitute(parameterValues[idx]), "");
+ namedParam.setParameterValue(parameters[idx], value);
+ } else {
+ // something filled in, in the field column...
+ //
+ String value = "";
+ if (resultRow != null) {
+ value = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ namedParam.setParameterValue(parameters[idx], value);
+ }
+ }
+ }
+ }
+
+ first = false;
+
+ Result previousResult = result;
+
+ try {
+ if (isDetailed()) {
+ logDetailed(BaseMessages.getString(
+ PKG, "JobTrans.StartingTrans", getFilename(), getName(), getDescription()));
+ }
+
+ if (clearResultRows) {
+ previousResult.setRows(new ArrayList());
+ }
+
+ if (clearResultFiles) {
+ previousResult.getResultFiles().clear();
+ }
+
+ /*
+ * Set one or more "result" rows on the transformation...
+ */
+ if (execPerRow) {
+ // Execute for each input row
+
+ if (argFromPrevious) {
+ // Copy the input row to the (command line) arguments
+
+ args = null;
+ if (resultRow != null) {
+ args = new String[resultRow.size()];
+ for (int i = 0; i < resultRow.size(); i++) {
+ args[i] = resultRow.getString(i, null);
+ }
+ }
+ } else {
+ // Just pass a single row
+ List newList = new ArrayList();
+ newList.add(resultRow);
+
+ // This previous result rows list can be either empty or not.
+ // Depending on the checkbox "clear result rows"
+ // In this case, it would execute the transformation with one extra row each time
+ // Can't figure out a real use-case for it, but hey, who am I to decide that, right?
+ // :-)
+ //
+ previousResult.getRows().addAll(newList);
+ }
+
+ if (paramsFromPrevious) { // Copy the input the parameters
+
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+ // We have a parameter
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ namedParam.setParameterValue(parameters[idx], Const.NVL(
+ environmentSubstitute(parameterValues[idx]), ""));
+ } else {
+ String fieldValue = "";
+
+ if (resultRow != null) {
+ fieldValue = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ // Get the value from the input stream
+ namedParam.setParameterValue(parameters[idx], Const.NVL(fieldValue, ""));
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (argFromPrevious) {
+ // Only put the first Row on the arguments
+ args = null;
+ if (resultRow != null) {
+ args = new String[resultRow.size()];
+ for (int i = 0; i < resultRow.size(); i++) {
+ args[i] = resultRow.getString(i, null);
+ }
+ }
+ }
+
+ if (paramsFromPrevious) {
+ // Copy the input the parameters
+ if (parameters != null) {
+ for (int idx = 0; idx < parameters.length; idx++) {
+ if (!Const.isEmpty(parameters[idx])) {
+ // We have a parameter
+ if (Const.isEmpty(Const.trim(parameterFieldNames[idx]))) {
+ namedParam.setParameterValue(parameters[idx], Const.NVL(
+ environmentSubstitute(parameterValues[idx]), ""));
+ } else {
+ String fieldValue = "";
+
+ if (resultRow != null) {
+ fieldValue = resultRow.getString(parameterFieldNames[idx], "");
+ }
+ // Get the value from the input stream
+ namedParam.setParameterValue(parameters[idx], Const.NVL(fieldValue, ""));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Handle the parameters...
+ //
+ transMeta.clearParameters();
+ String[] parameterNames = transMeta.listParameters();
+ for (int idx = 0; idx < parameterNames.length; idx++) {
+ // Grab the parameter value set in the Trans job entry
+ //
+ String thisValue = namedParam.getParameterValue(parameterNames[idx]);
+ if (!Const.isEmpty(thisValue)) {
+ // Set the value as specified by the user in the job entry
+ //
+ transMeta.setParameterValue(parameterNames[idx], thisValue);
+ } else {
+ // See if the parameter had a value set in the parent job...
+ // This value should pass down to the transformation if that's what we opted to do.
+ //
+ if (isPassingAllParameters()) {
+ String parentValue = parentJob.getParameterValue(parameterNames[idx]);
+ if (!Const.isEmpty(parentValue)) {
+ transMeta.setParameterValue(parameterNames[idx], parentValue);
+ }
+ }
+ }
+ }
+
+ // Execute this transformation across a cluster of servers
+ //
+ if (clustering) {
+ TransExecutionConfiguration executionConfiguration = new TransExecutionConfiguration();
+ executionConfiguration.setClusterPosting(true);
+ executionConfiguration.setClusterPreparing(true);
+ executionConfiguration.setClusterStarting(true);
+ executionConfiguration.setClusterShowingTransformation(false);
+ executionConfiguration.setSafeModeEnabled(false);
+ executionConfiguration.setRepository(rep);
+ executionConfiguration.setLogLevel(transLogLevel);
+ executionConfiguration.setPreviousResult(previousResult);
+
+ // Also pass the variables from the transformation into the execution configuration
+ // That way it can go over the HTTP connection to the slave server.
+ //
+ executionConfiguration.setVariables(transMeta);
+
+ // Also set the arguments...
+ //
+ executionConfiguration.setArgumentStrings(args);
+
+ if (parentJob.getJobMeta().isBatchIdPassed()) {
+ executionConfiguration.setPassedBatchId(parentJob.getPassedBatchId());
+ }
+
+ TransSplitter transSplitter = null;
+ long errors = 0;
+ try {
+ transSplitter = Trans.executeClustered(transMeta, executionConfiguration);
+
+ // Monitor the running transformations, wait until they are done.
+ // Also kill them all if anything goes bad
+ // Also clean up afterwards...
+ //
+ errors += Trans.monitorClusteredTransformation(log, transSplitter, parentJob);
+
+ } catch (Exception e) {
+ logError("Error during clustered execution. Cleaning up clustered execution.", e);
+ // In case something goes wrong, make sure to clean up afterwards!
+ //
+ errors++;
+ if (transSplitter != null) {
+ Trans.cleanupCluster(log, transSplitter);
+ } else {
+ // Try to clean anyway...
+ //
+ SlaveServer master = null;
+ for (StepMeta stepMeta : transMeta.getSteps()) {
+ if (stepMeta.isClustered()) {
+ for (SlaveServer slaveServer : stepMeta.getClusterSchema().getSlaveServers()) {
+ if (slaveServer.isMaster()) {
+ master = slaveServer;
+ break;
+ }
+ }
+ }
+ }
+ if (master != null) {
+ master.deAllocateServerSockets(transMeta.getName(), null);
+ }
+ }
+ }
+
+ result.clear();
+
+ if (transSplitter != null) {
+ Result clusterResult =
+ Trans.getClusteredTransformationResult(log, transSplitter, parentJob, loggingRemoteWork);
+ result.add(clusterResult);
+ }
+
+ result.setNrErrors(result.getNrErrors() + errors);
+
+ } else if (remoteSlaveServer != null) {
+ // Execute this transformation remotely
+ //
+
+ // Make sure we can parameterize the slave server connection
+ //
+ remoteSlaveServer.shareVariablesWith(this);
+
+ // Remote execution...
+ //
+ TransExecutionConfiguration transExecutionConfiguration = new TransExecutionConfiguration();
+ transExecutionConfiguration.setPreviousResult(previousResult.clone());
+ transExecutionConfiguration.setArgumentStrings(args);
+ transExecutionConfiguration.setVariables(this);
+ transExecutionConfiguration.setRemoteServer(remoteSlaveServer);
+ transExecutionConfiguration.setLogLevel(transLogLevel);
+ transExecutionConfiguration.setRepository(rep);
+ transExecutionConfiguration.setLogFileName(realLogFilename);
+ transExecutionConfiguration.setSetAppendLogfile(setAppendLogfile);
+ transExecutionConfiguration.setSetLogfile(setLogfile);
+ // FIXME hard-coded option as there's no way to declare passingExport for Trans Step like Job
+ transExecutionConfiguration.setPassingExport(true);
+
+ Map params = transExecutionConfiguration.getParams();
+ for (String param : transMeta.listParameters()) {
+ String value =
+ Const.NVL(transMeta.getParameterValue(param), Const.NVL(
+ transMeta.getParameterDefault(param), transMeta.getVariable(param)));
+ params.put(param, value);
+ }
+
+ if (parentJob.getJobMeta().isBatchIdPassed()) {
+ transExecutionConfiguration.setPassedBatchId(parentJob.getPassedBatchId());
+ }
+
+ // Send the XML over to the slave server
+ // Also start the transformation over there...
+ //
+ String carteObjectId = Trans.sendToSlaveServer(transMeta, transExecutionConfiguration, rep, metaStore);
+
+ // Now start the monitoring...
+ //
+ SlaveServerTransStatus transStatus = null;
+ RemoteJobEntryLogHelper logHelper = new RemoteJobEntryLogHelper(
+ remoteSlaveServer, carteObjectId, this.log);
+ while (!parentJob.isStopped() && waitingToFinish) {
+ try {
+ transStatus = remoteSlaveServer.getTransStatus(transMeta.getName(), carteObjectId,
+ logHelper.getLastLogEntryNo());
+ logHelper.log(transStatus.getLoggingString(),
+ transStatus.getFirstLoggingLineNr(), transStatus.getLastLoggingLineNr());
+ if (!transStatus.isRunning()) {
+ // The transformation is finished, get the result...
+ //
+ Result remoteResult = transStatus.getResult();
+ result.clear();
+ result.add(remoteResult);
+
+ // In case you manually stop the remote trans (browser etc), make sure it's marked as an error
+ //
+ if (remoteResult.isStopped()) {
+ result.setNrErrors(result.getNrErrors() + 1); //
+ }
+
+ // Make sure to clean up : write a log record etc, close any left-over sockets etc.
+ //
+ remoteSlaveServer.cleanupTransformation(transMeta.getName(), carteObjectId);
+
+ break;
+ }
+ } catch (Exception e1) {
+
+ logError(BaseMessages.getString(PKG, "JobTrans.Error.UnableContactSlaveServer", ""
+ + remoteSlaveServer, transMeta.getName()), e1);
+ result.setNrErrors(result.getNrErrors() + 1L);
+ break; // Stop looking too, chances are too low the server will come back on-line
+ }
+
+ // sleep for 2 seconds
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+
+ if (parentJob.isStopped()) {
+ // See if we have a status and if we need to stop the remote execution here...
+ //
+ if (transStatus == null || transStatus.isRunning()) {
+ // Try a remote abort ...
+ //
+ remoteSlaveServer.stopTransformation(transMeta.getName(), transStatus.getId());
+
+ // And a cleanup...
+ //
+ remoteSlaveServer.cleanupTransformation(transMeta.getName(), transStatus.getId());
+
+ // Set an error state!
+ //
+ result.setNrErrors(result.getNrErrors() + 1L);
+ }
+ }
+
+ } else {
+
+ // Execute this transformation on the local machine
+ //
+
+ // Create the transformation from meta-data
+ //
+ trans = new Trans(transMeta, this);
+
+ // Pass the socket repository as early as possible...
+ //
+ trans.setSocketRepository(parentJob.getSocketRepository());
+
+ if (parentJob.getJobMeta().isBatchIdPassed()) {
+ trans.setPassedBatchId(parentJob.getPassedBatchId());
+ }
+
+ // set the parent job on the transformation, variables are taken from here...
+ //
+ trans.setParentJob(parentJob);
+ trans.setParentVariableSpace(parentJob);
+ trans.setLogLevel(transLogLevel);
+ trans.setPreviousResult(previousResult);
+ trans.setArguments(arguments);
+
+ // Mappings need the repository to load from
+ //
+ trans.setRepository(rep);
+
+ // inject the metaStore
+ trans.setMetaStore(metaStore);
+
+ // First get the root job
+ //
+ Job rootJob = parentJob;
+ while (rootJob.getParentJob() != null) {
+ rootJob = rootJob.getParentJob();
+ }
+
+ // Get the start and end-date from the root job...
+ //
+ trans.setJobStartDate(rootJob.getStartDate());
+ trans.setJobEndDate(rootJob.getEndDate());
+
+ // Inform the parent job we started something here...
+ //
+ for (DelegationListener delegationListener : parentJob.getDelegationListeners()) {
+ // TODO: copy some settings in the job execution configuration, not strictly needed
+ // but the execution configuration information is useful in case of a job re-start
+ //
+ delegationListener.transformationDelegationStarted(trans, new TransExecutionConfiguration());
+ }
+
+ try {
+ // Start execution...
+ //
+ trans.execute(args);
+
+ // Wait until we're done with it...
+ //TODO is it possible to implement Observer pattern to avoid Thread.sleep here?
+ while (!trans.isFinished() && trans.getErrors() == 0) {
+ if (parentJob.isStopped()) {
+ trans.stopAll();
+ break;
+ } else {
+ try {
+ Thread.sleep(0, 500);
+ } catch (InterruptedException e) {
+ // Ignore errors
+ }
+ }
+ }
+ trans.waitUntilFinished();
+
+ if (parentJob.isStopped() || trans.getErrors() != 0) {
+ trans.stopAll();
+ result.setNrErrors(1);
+ }
+ Result newResult = trans.getResult();
+
+ result.clear(); // clear only the numbers, NOT the files or rows.
+ result.add(newResult);
+
+ // Set the result rows too, if any ...
+ if (!Const.isEmpty(newResult.getRows())) {
+ result.setRows(newResult.getRows());
+ }
+
+ if (setLogfile) {
+ ResultFile resultFile =
+ new ResultFile(
+ ResultFile.FILE_TYPE_LOG, KettleVFS.getFileObject(realLogFilename, this), parentJob
+ .getJobname(), toString()
+ );
+ result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
+ }
+ } catch (KettleException e) {
+
+ logError(BaseMessages.getString(PKG, "JobTrans.Error.UnablePrepareExec"), e);
+ result.setNrErrors(1);
+ }
+ }
+ } catch (Exception e) {
+
+ logError(BaseMessages.getString(PKG, "JobTrans.ErrorUnableOpenTrans", e.getMessage()));
+ logError(Const.getStackTracker(e));
+ result.setNrErrors(1);
+ }
+ iteration++;
+ }
+
+ if (setLogfile) {
+ if (logChannelFileWriter != null) {
+ logChannelFileWriter.stopLogging();
+
+ ResultFile resultFile =
+ new ResultFile(
+ ResultFile.FILE_TYPE_LOG, logChannelFileWriter.getLogFile(), parentJob.getJobname(), getName());
+ result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
+
+ // See if anything went wrong during file writing...
+ //
+ if (logChannelFileWriter.getException() != null) {
+ logError("Unable to open log file [" + getLogFilename() + "] : ");
+ logError(Const.getStackTracker(logChannelFileWriter.getException()));
+ result.setNrErrors(1);
+ result.setResult(false);
+ return result;
+ }
+ }
+ }
+
+ if (result.getNrErrors() == 0) {
+ result.setResult(true);
+ } else {
+ result.setResult(false);
+ }
+
+ return result;
+ }
+
+ @Deprecated
+ public TransMeta getTransMeta(Repository rep, VariableSpace space) throws KettleException {
+ return getTransMeta(rep, null, space);
+ }
+
+ public TransMeta getTransMeta(Repository rep, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ try {
+ TransMeta transMeta = null;
+ CurrentDirectoryResolver r = new CurrentDirectoryResolver();
+ VariableSpace tmpSpace = r.resolveCurrentDirectory(
+ specificationMethod, space, rep, parentJob, getFilename());
+ switch (specificationMethod) {
+ case FILENAME:
+ String realFilename = tmpSpace.environmentSubstitute(getFilename());
+ if (rep != null) {
+ realFilename = r.normalizeSlashes(realFilename);
+ // need to try to load from the repository
+ try {
+ String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
+ String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1);
+ RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
+ // transMeta = rep.loadTransformation(tmpFilename, dir, null, true, null);
+ transMeta = ResourceDefinitionHelper.loadTransformation(rep, dir, tmpFilename);
+ } catch (KettleException ke) {
+ // try without extension
+ if (realFilename.endsWith(Const.STRING_TRANS_DEFAULT_EXT)) {
+ try {
+ String tmpFilename = realFilename.substring(realFilename.lastIndexOf("/") + 1,
+ realFilename.indexOf("." + Const.STRING_TRANS_DEFAULT_EXT));
+ String dirStr = realFilename.substring(0, realFilename.lastIndexOf("/"));
+ RepositoryDirectoryInterface dir = rep.findDirectory(dirStr);
+ transMeta = rep.loadTransformation(tmpFilename, dir, null, true, null);
+ } catch (KettleException ke2) {
+ // fall back to try loading from file system (transMeta is going to be null)
+ }
+ }
+ }
+ }
+ if (transMeta == null) {
+ logBasic("Loading transformation from XML file [" + realFilename + "]");
+ transMeta = new TransMeta(realFilename, metaStore, null, true, this, null);
+ }
+ break;
+ case REPOSITORY_BY_NAME:
+ String transname = tmpSpace.environmentSubstitute(getTransname());
+ String realDirectory = tmpSpace.environmentSubstitute(getDirectory());
+
+ logBasic(BaseMessages.getString(PKG, "JobTrans.Log.LoadingTransRepDirec", transname, realDirectory));
+
+ if (rep != null) {
+ //
+ // It only makes sense to try to load from the repository when the
+ // repository is also filled in.
+ //
+ // It reads last the last revision from the repository.
+ //
+ realDirectory = r.normalizeSlashes(realDirectory);
+
+ RepositoryDirectoryInterface repositoryDirectory = rep.findDirectory(realDirectory);
+ transMeta = rep.loadTransformation(transname, repositoryDirectory, null, true, null);
+ } else {
+ // rep is null, let's try loading by filename
+ try {
+ transMeta = new TransMeta(realDirectory + "/" + transname, metaStore, null, true, this, null);
+ } catch (KettleException ke) {
+ try {
+ // add .ktr extension and try again
+ transMeta = new TransMeta(realDirectory + "/" + transname + "." + Const.STRING_TRANS_DEFAULT_EXT,
+ metaStore, null, true, this, null);
+ } catch (KettleException ke2) {
+ throw new KettleException(BaseMessages.getString(PKG, "JobTrans.Exception.NoRepDefined"), ke2);
+ }
+ }
+ }
+ break;
+ case REPOSITORY_BY_REFERENCE:
+ if (transObjectId == null) {
+ throw new KettleException(BaseMessages.getString(PKG,
+ "JobTrans.Exception.ReferencedTransformationIdIsNull"));
+ }
+
+ if (rep != null) {
+ // Load the last revision
+ //
+ transMeta = rep.loadTransformation(transObjectId, null);
+ }
+ break;
+ default:
+ throw new KettleException("The specified object location specification method '"
+ + specificationMethod + "' is not yet supported in this job entry.");
+ }
+
+ if (transMeta != null) {
+ // copy parent variables to this loaded variable space.
+ //
+ transMeta.copyVariablesFrom(this);
+
+ // Pass repository and metastore references
+ //
+ transMeta.setRepository(rep);
+ transMeta.setMetaStore(metaStore);
+ }
+
+ return transMeta;
+ } catch (Exception e) {
+ throw new KettleException(BaseMessages.getString(PKG, "JobTrans.Exception.MetaDataLoad"), e);
+ }
+ }
+
+ public boolean evaluates() {
+ return true;
+ }
+
+ public boolean isUnconditional() {
+ return true;
+ }
+
+ public List getSQLStatements(Repository repository, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ this.copyVariablesFrom(space);
+ TransMeta transMeta = getTransMeta(repository, metaStore, this);
+
+ return transMeta.getSQLStatements();
+ }
+
+ /**
+ * @return Returns the directoryPath.
+ */
+ public String getDirectoryPath() {
+ return directoryPath;
+ }
+
+ /**
+ * @param directoryPath The directoryPath to set.
+ */
+ public void setDirectoryPath(String directoryPath) {
+ this.directoryPath = directoryPath;
+ }
+
+ /**
+ * @return the clustering
+ */
+ public boolean isClustering() {
+ return clustering;
+ }
+
+ /**
+ * @param clustering the clustering to set
+ */
+ public void setClustering(boolean clustering) {
+ this.clustering = clustering;
+ }
+
+ public void check(List remarks, JobMeta jobMeta, VariableSpace space,
+ Repository repository, IMetaStore metaStore) {
+ if (setLogfile) {
+ JobEntryValidatorUtils.andValidator().validate(this, "logfile", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ }
+ if (!Const.isEmpty(filename)) {
+ JobEntryValidatorUtils.andValidator().validate(this, "filename", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ } else {
+ JobEntryValidatorUtils.andValidator().validate(this, "transname", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notBlankValidator()));
+ JobEntryValidatorUtils.andValidator().validate(this, "directory", remarks, AndValidator.putValidators(JobEntryValidatorUtils.notNullValidator()));
+ }
+ }
+
+ public List getResourceDependencies(JobMeta jobMeta) {
+ List references = super.getResourceDependencies(jobMeta);
+ if (!Const.isEmpty(filename)) {
+ // During this phase, the variable space hasn't been initialized yet - it seems
+ // to happen during the execute. As such, we need to use the job meta's resolution
+ // of the variables.
+ String realFileName = jobMeta.environmentSubstitute(filename);
+ ResourceReference reference = new ResourceReference(this);
+ reference.getEntries().add(new ResourceEntry(realFileName, ResourceType.ACTIONFILE));
+ references.add(reference);
+ }
+ return references;
+ }
+
+ /**
+ * We're going to load the transformation meta data referenced here. Then we're going to give it a new filename,
+ * modify that filename in this entries. The parent caller will have made a copy of it, so it should be OK to do so.
+ *
+ * Exports the object to a flat-file system, adding content with filename keys to a set of definitions. The supplied
+ * resource naming interface allows the object to name appropriately without worrying about those parts of the
+ * implementation specific details.
+ *
+ * @param space The variable space to resolve (environment) variables with.
+ * @param definitions The map containing the filenames and content
+ * @param namingInterface The resource naming interface allows the object to be named appropriately
+ * @param repository The repository to load resources from
+ * @param metaStore the metaStore to load external metadata from
+ * @return The filename for this object. (also contained in the definitions map)
+ * @throws KettleException in case something goes wrong during the export
+ */
+ public String exportResources(VariableSpace space, Map definitions,
+ ResourceNamingInterface namingInterface, Repository repository, IMetaStore metaStore) throws KettleException {
+ // Try to load the transformation from repository or file.
+ // Modify this recursively too...
+ //
+ // AGAIN: there is no need to clone this job entry because the caller is responsible for this.
+ //
+ // First load the transformation metadata...
+ //
+ copyVariablesFrom(space);
+ TransMeta transMeta = getTransMeta(repository, space);
+
+ if (transMeta instanceof ResourceDefinitionHelper.TransMetaCollection) {
+ ResourceDefinitionHelper.TransMetaCollection tmc = (ResourceDefinitionHelper.TransMetaCollection) transMeta;
+ for (TransMeta t : tmc.getAttachedMeta()) {
+ if (!ResourceDefinitionHelper.containsResource(definitions, space, namingInterface, t)) {
+ t.exportResources(transMeta, definitions, namingInterface, repository, metaStore);
+ }
+ }
+
+ return null;
+ }
+
+ // Also go down into the transformation and export the files there. (mapping recursively down)
+ //
+ String proposedNewFilename =
+ transMeta.exportResources(transMeta, definitions, namingInterface, repository, metaStore);
+
+ // To get a relative path to it, we inject ${Internal.Job.Filename.Directory}
+ //
+ String newFilename = "${" + Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY + "}/" + proposedNewFilename;
+
+ // Set the correct filename inside the XML.
+ //
+ transMeta.setFilename(newFilename);
+
+ // exports always reside in the root directory, in case we want to turn this into a file repository...
+ //
+ transMeta.setRepositoryDirectory(new RepositoryDirectory());
+
+ // export to filename ALWAYS (this allows the exported XML to be executed remotely)
+ //
+ setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
+
+ // change it in the job entry
+ //
+ filename = newFilename;
+
+ return proposedNewFilename;
+ }
+
+ protected String getLogfile() {
+ return logfile;
+ }
+
+ /**
+ * @return the remote slave server name
+ */
+ public String getRemoteSlaveServerName() {
+ return remoteSlaveServerName;
+ }
+
+ /**
+ * @param remoteSlaveServerName the remote slave server name to set
+ */
+ public void setRemoteSlaveServerName(String remoteSlaveServerName) {
+ this.remoteSlaveServerName = remoteSlaveServerName;
+ }
+
+ /**
+ * @return the waitingToFinish
+ */
+ public boolean isWaitingToFinish() {
+ return waitingToFinish;
+ }
+
+ /**
+ * @param waitingToFinish the waitingToFinish to set
+ */
+ public void setWaitingToFinish(boolean waitingToFinish) {
+ this.waitingToFinish = waitingToFinish;
+ }
+
+ /**
+ * @return the followingAbortRemotely
+ */
+ public boolean isFollowingAbortRemotely() {
+ return followingAbortRemotely;
+ }
+
+ /**
+ * @param followingAbortRemotely the followingAbortRemotely to set
+ */
+ public void setFollowingAbortRemotely(boolean followingAbortRemotely) {
+ this.followingAbortRemotely = followingAbortRemotely;
+ }
+
+ public boolean isLoggingRemoteWork() {
+ return loggingRemoteWork;
+ }
+
+ public void setLoggingRemoteWork(boolean loggingRemoteWork) {
+ this.loggingRemoteWork = loggingRemoteWork;
+ }
+
+ /**
+ * @return the passingAllParameters
+ */
+ public boolean isPassingAllParameters() {
+ return passingAllParameters;
+ }
+
+ /**
+ * @param passingAllParameters the passingAllParameters to set
+ */
+ public void setPassingAllParameters(boolean passingAllParameters) {
+ this.passingAllParameters = passingAllParameters;
+ }
+
+ public Trans getTrans() {
+ return trans;
+ }
+
+ /**
+ * @return the transObjectId
+ */
+ public ObjectId getTransObjectId() {
+ return transObjectId;
+ }
+
+ /**
+ * @param transObjectId the transObjectId to set
+ */
+ public void setTransObjectId(ObjectId transObjectId) {
+ this.transObjectId = transObjectId;
+ }
+
+ /**
+ * @return the specificationMethod
+ */
+ public ObjectLocationSpecificationMethod getSpecificationMethod() {
+ return specificationMethod;
+ }
+
+ /**
+ * @param specificationMethod the specificationMethod to set
+ */
+ public void setSpecificationMethod(ObjectLocationSpecificationMethod specificationMethod) {
+ this.specificationMethod = specificationMethod;
+ }
+
+ public boolean hasRepositoryReferences() {
+ return specificationMethod == ObjectLocationSpecificationMethod.REPOSITORY_BY_REFERENCE;
+ }
+
+ /**
+ * Look up the references after import
+ *
+ * @param repository the repository to reference.
+ */
+ public void lookupRepositoryReferences(Repository repository) throws KettleException {
+ // The correct reference is stored in the trans name and directory attributes...
+ //
+ RepositoryDirectoryInterface repositoryDirectoryInterface =
+ RepositoryImportLocation.getRepositoryImportLocation().findDirectory(directory);
+ transObjectId = repository.getTransformationID(transname, repositoryDirectoryInterface);
+ }
+
+ /**
+ * @return The objects referenced in the step, like a a transformation, a job, a mapper, a reducer, a combiner, ...
+ */
+ public String[] getReferencedObjectDescriptions() {
+ return new String[]{BaseMessages.getString(PKG, "JobEntryTrans.ReferencedObject.Description"),};
+ }
+
+ private boolean isTransformationDefined() {
+ return !Const.isEmpty(filename)
+ || transObjectId != null || (!Const.isEmpty(this.directory) && !Const.isEmpty(transname));
+ }
+
+ public boolean[] isReferencedObjectEnabled() {
+ return new boolean[]{isTransformationDefined(),};
+ }
+
+ /**
+ * Load the referenced object
+ *
+ * @param index the referenced object index to load (in case there are multiple references)
+ * @param rep the repository
+ * @param metaStore metaStore
+ * @param space the variable space to use
+ * @return the referenced object once loaded
+ * @throws KettleException
+ */
+ public Object loadReferencedObject(int index, Repository rep, IMetaStore metaStore, VariableSpace space) throws KettleException {
+ return getTransMeta(rep, metaStore, space);
+ }
+
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/repository/pur/LazyUnifiedRepositoryDirectory.java b/pentaho-kettle/src/main/java/org/pentaho/di/repository/pur/LazyUnifiedRepositoryDirectory.java
new file mode 100644
index 0000000..d8cc6eb
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/repository/pur/LazyUnifiedRepositoryDirectory.java
@@ -0,0 +1,367 @@
+/*!
+ * Copyright 2016 Pentaho Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.pentaho.di.repository.pur;
+
+import org.apache.commons.lang.StringUtils;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.repository.*;
+import org.pentaho.di.repository.pur.model.EERepositoryObject;
+import org.pentaho.di.repository.pur.model.RepositoryLock;
+import org.pentaho.di.ui.repository.pur.services.ILockService;
+import org.pentaho.platform.api.repository2.unified.IUnifiedRepository;
+import org.pentaho.platform.api.repository2.unified.RepositoryFile;
+import org.pentaho.platform.api.repository2.unified.RepositoryFileTree;
+import org.pentaho.platform.api.repository2.unified.RepositoryRequest;
+import org.pentaho.platform.repository2.ClientRepositoryPaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.pentaho.di.repository.pur.PurRepository.getObjectType;
+
+/**
+ * A version of RepositoryDirectoryInterface which only loads from the underlying repository as needed (Lazy)
+ *
+ * Created by nbaker on 12/22/15.
+ *
+ * Note the only reason we're extending RepositoryDirectory instead of implementing RepositoryDirectoryInterface is due
+ * to some interface methods returning RepositoryDirectory!!
+ */
+public class LazyUnifiedRepositoryDirectory extends RepositoryDirectory {
+
+ private RepositoryFile self;
+ private IUnifiedRepository repository;
+ private RepositoryServiceRegistry registry;
+ private List subdirectories;
+ private List fileChildren;
+ private RepositoryDirectoryInterface parent;
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ public LazyUnifiedRepositoryDirectory(RepositoryFile self, RepositoryDirectoryInterface parent,
+ IUnifiedRepository repository, RepositoryServiceRegistry registry) {
+ this.self = self;
+ this.parent = parent;
+ this.repository = repository;
+ this.registry = registry;
+ }
+
+ private String getParentPath(String absolutePath) {
+ int parentEndIndex;
+ if (absolutePath.endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ parentEndIndex = absolutePath.lastIndexOf(RepositoryDirectory.DIRECTORY_SEPARATOR, absolutePath.length() - 2);
+ } else {
+ parentEndIndex = absolutePath.lastIndexOf(RepositoryDirectory.DIRECTORY_SEPARATOR);
+ }
+ if (parentEndIndex < 0) {
+ return null;
+ }
+ return absolutePath.substring(0, parentEndIndex);
+ }
+
+ @Override
+ public RepositoryDirectory findDirectory(String path) {
+ if (StringUtils.isEmpty(path)) {
+ return null;
+ }
+ String absolutePath;
+ if (path.startsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ if (self.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ absolutePath = self.getPath() + path.substring(1);
+ } else {
+ absolutePath = self.getPath() + path;
+ }
+ } else {
+ if (self.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ absolutePath = self.getPath() + path;
+ } else {
+ absolutePath = self.getPath() + RepositoryDirectory.DIRECTORY_SEPARATOR + path;
+ }
+ }
+
+ RepositoryFile file = repository.getFile(absolutePath);
+ if (file == null || !file.isFolder()) {
+ return null;
+ }
+ if (isRoot() && RepositoryDirectory.DIRECTORY_SEPARATOR.equals(absolutePath)) {
+ return this;
+ }
+
+ // Verifies if this is the parent directory of file and if so passes this as parent argument
+ String parentPath = getParentPath(absolutePath);
+ if (self.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR)) {
+ if (parentPath.equals(self.getPath().substring(0, self.getPath().length() - 1))) {
+ return new LazyUnifiedRepositoryDirectory(file, this, repository, registry);
+ }
+ } else {
+ if (parentPath.equals(self.getPath())) {
+ return new LazyUnifiedRepositoryDirectory(file, this, repository, registry);
+ }
+ }
+
+ return new LazyUnifiedRepositoryDirectory(file, findDirectory(parentPath), repository, registry);
+
+ }
+
+ @Override
+ public RepositoryDirectory findChild(String name) {
+ return findDirectory(name);
+ }
+
+ @Override
+ public RepositoryDirectory findDirectory(String[] path) {
+ return findDirectory(StringUtils.join(path, "/"));
+ }
+
+ @Override
+ public List getChildren() {
+ if (subdirectories == null) {
+ subdirectories = new ArrayList<>();
+ synchronized (subdirectories) {
+ List children = getAllURChildrenFiles();
+ for (RepositoryFile child : children) {
+ LazyUnifiedRepositoryDirectory dir = new LazyUnifiedRepositoryDirectory(child, this, repository, registry);
+ dir.setObjectId(new StringObjectId(child.getId().toString()));
+ this.addSubdirectory(dir);
+ }
+ }
+ }
+ return subdirectories;
+ }
+
+ @Override
+ public List getRepositoryObjects() {
+ if (fileChildren == null) {
+
+ fileChildren = new ArrayList();
+ synchronized (fileChildren) {
+
+ UnifiedRepositoryLockService lockService =
+ (UnifiedRepositoryLockService) registry.getService(ILockService.class);
+
+ RepositoryFileTree tree = repository.getTree(new RepositoryRequest(this.self.getPath(), true, 1, null));
+
+ for (RepositoryFileTree tchild : tree.getChildren()) {
+ RepositoryFile child = tchild.getFile();
+
+ RepositoryLock lock = null;
+ try {
+ // No idea why introducing the unnecessary dependency and NPE here...
+ lock = lockService == null ? null : lockService.getLock(child);
+ RepositoryObjectType objectType = getObjectType(child.getName());
+ EERepositoryObject repositoryObject =
+ new EERepositoryObject(child, this, null, objectType, null, lock, false);
+
+ repositoryObject.setVersioningEnabled(tchild.getVersioningEnabled());
+ repositoryObject.setVersionCommentEnabled(tchild.getVersionCommentEnabled());
+ fileChildren.add(repositoryObject);
+ } catch (KettleException e) {
+ logger.error("Error converting Unified Repository file to PDI RepositoryObject: " + child.getPath()
+ + ". File will be skipped", e);
+ }
+ }
+ }
+ }
+ return fileChildren;
+
+ }
+
+ @Override
+ public void setRepositoryObjects(List list) {
+ synchronized (fileChildren) {
+ fileChildren.clear();
+ fileChildren.addAll(list);
+ }
+ }
+
+ @Override
+ public boolean isVisible() {
+ return !isRoot() && !self.isHidden();
+ }
+
+
+ @Override
+ public int getNrSubdirectories() {
+ List childrenFiles = getAllURChildrenFiles();
+ return childrenFiles.size();
+ }
+
+ @Override
+ public RepositoryDirectory getSubdirectory(int i) {
+ if (subdirectories == null) {
+ getChildren();
+ }
+
+ if (i >= subdirectories.size() || i < 0) {
+ return null;
+ }
+ RepositoryDirectoryInterface directoryInterface = subdirectories.get(i);
+ // Have to cast due to bad interface
+ if (directoryInterface instanceof RepositoryDirectory) {
+ return (RepositoryDirectory) directoryInterface;
+ }
+ throw new IllegalStateException(
+ "Bad Repository interface expects RepositoryDirectoryInterface to be an instance of"
+ + " RepositoryDirectory. This class is not: " + directoryInterface.getClass().getName());
+ }
+
+ private List getAllURChildrenFiles() {
+ RepositoryRequest repositoryRequest = new RepositoryRequest();
+ repositoryRequest.setShowHidden(true);
+ repositoryRequest.setTypes(RepositoryRequest.FILES_TYPE_FILTER.FOLDERS);
+ repositoryRequest.setPath(this.self.getId().toString());
+ List children = repository.getChildren(repositoryRequest);
+
+
+ // Special case: /etc should not be returned from a directory listing.
+ RepositoryFile etcFile = null;
+ if (this.isRoot()) {
+ etcFile = repository.getFile(ClientRepositoryPaths.getEtcFolderPath());
+ }
+
+ // Filter for Folders only doesn't appear to work
+ Iterator iterator = children.iterator();
+ while (iterator.hasNext()) {
+ RepositoryFile next = iterator.next();
+ if (!next.isFolder()) {
+ iterator.remove();
+ }
+
+ // Special case: /etc should not be returned from a directory listing.
+ if (this.isRoot() && next.equals(etcFile)) {
+ iterator.remove();
+ }
+ }
+ return children;
+ }
+
+ @Override
+ public void clear() {
+ if (this.fileChildren != null) {
+ synchronized (fileChildren) {
+ this.fileChildren.clear();
+ }
+ }
+ if (this.subdirectories != null) {
+ synchronized (subdirectories) {
+ this.subdirectories.clear();
+ }
+ }
+ }
+
+ @Override
+ public void addSubdirectory(RepositoryDirectoryInterface repositoryDirectoryInterface) {
+ if (subdirectories == null) {
+ subdirectories = new ArrayList<>();
+ }
+ synchronized (subdirectories) {
+ this.subdirectories.add(repositoryDirectoryInterface);
+ }
+ }
+
+ @Override
+ public String getName() {
+ return self.getName();
+ }
+
+ @Override
+ public String getPath() {
+ return self.getPath();
+ }
+
+ @Override
+ public ObjectId getObjectId() {
+ return new StringObjectId(self.getId().toString());
+ }
+
+ @Override
+ public void setChildren(List list) {
+ if (subdirectories == null) {
+ subdirectories = new ArrayList<>();
+ }
+ if (!subdirectories.equals(list)) {
+ synchronized (subdirectories) {
+ subdirectories.clear();
+ subdirectories.addAll(list);
+ }
+ }
+ }
+
+ @Override
+ public String[] getPathArray() {
+ return getPath().split(RepositoryDirectory.DIRECTORY_SEPARATOR);
+ }
+
+
+ @Override
+ public ObjectId[] getDirectoryIDs() {
+ List children = this.getAllURChildrenFiles();
+ ObjectId[] objectIds = new ObjectId[children.size()];
+ for (int i = 0; i < children.size(); i++) {
+ objectIds[i] = new StringObjectId(children.get(i).getId().toString());
+ }
+
+ return objectIds;
+ }
+
+ @Override
+ public boolean isRoot() {
+ return parent == null;
+ }
+
+ @Override
+ public RepositoryDirectoryInterface findRoot() {
+ RepositoryDirectoryInterface current = this;
+ RepositoryDirectoryInterface parent = null;
+ while ((parent = current.getParent()) != null) {
+ current = parent;
+ }
+ return current;
+ }
+
+ @Override
+ public void setParent(RepositoryDirectoryInterface repositoryDirectoryInterface) {
+ this.parent = repositoryDirectoryInterface;
+ }
+
+ @Override
+ public RepositoryDirectoryInterface getParent() {
+ return parent;
+ }
+
+ @Override
+ public void setObjectId(ObjectId objectId) {
+ // ignore
+ }
+
+ @Override
+ public void setName(String s) {
+ // ignore
+ }
+
+
+ @Override
+ public String getPathObjectCombination(String transName) {
+ if (isRoot()) {
+ return getPath() + transName;
+ } else {
+ return getPath() + RepositoryDirectory.DIRECTORY_SEPARATOR + transName;
+ }
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceDefinitionHelper.java b/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceDefinitionHelper.java
new file mode 100644
index 0000000..cf320d1
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceDefinitionHelper.java
@@ -0,0 +1,179 @@
+/*! ******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+package org.pentaho.di.resource;
+
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.base.AbstractMeta;
+import org.pentaho.di.core.Const;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.exception.KettleFileException;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.JobMeta;
+import org.pentaho.di.repository.*;
+import org.pentaho.di.trans.TransMeta;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility class for loading job and transformation meta data.
+ *
+ * @author Zhichun Wu
+ */
+public final class ResourceDefinitionHelper {
+ private final static Class PKG = JobMeta.class; // for i18n purposes, needed by Translator2!!
+
+ private final static String VARIABLE_PREFIX = "${";
+ private final static String VARIABLE_SUFFIX = "}";
+
+ public static class TransMetaCollection extends TransMeta {
+ private final List attachedMeta = new ArrayList<>();
+
+ public void attachTransMeta(TransMeta transMeta) {
+ attachedMeta.add(transMeta);
+ }
+
+ public List getAttachedMeta() {
+ return attachedMeta;
+ }
+ }
+
+ public static class JobMetaCollection extends JobMeta {
+ private final List attachedMeta = new ArrayList<>();
+
+ public void attachJobMeta(JobMeta jobMeta) {
+ attachedMeta.add(jobMeta);
+ }
+
+ public List getAttachedMeta() {
+ return attachedMeta;
+ }
+ }
+
+ public static boolean containsVariable(String name) {
+ boolean hasVar = false;
+
+ if (name != null) {
+ int index = name.indexOf(VARIABLE_PREFIX);
+ // variable name should at least contain one character
+ index = index >= 0 ? name.indexOf(VARIABLE_SUFFIX, index + VARIABLE_PREFIX.length() + 1) : -1;
+
+ hasVar = index > 0;
+ }
+
+ return hasVar;
+ }
+
+ public static boolean containsResource(Map definitions,
+ VariableSpace space,
+ ResourceNamingInterface namingInterface,
+ AbstractMeta meta) throws KettleException {
+ if (definitions == null || space == null || namingInterface == null || meta == null) {
+ return false;
+ }
+
+ // String baseName;
+ // String originalPath;
+ String extension = meta instanceof TransMeta ? Const.STRING_TRANS_DEFAULT_EXT : Const.STRING_JOB_DEFAULT_EXT;
+ String fullname;
+ try {
+ RepositoryDirectoryInterface directory = meta.getRepositoryDirectory();
+ if (Const.isEmpty(meta.getFilename())) {
+ // Assume repository...
+ //
+ // originalPath = directory.getPath();
+ // baseName = meta.getName();
+ fullname =
+ directory.getPath() + (directory.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR) ? ""
+ : RepositoryDirectory.DIRECTORY_SEPARATOR) + meta.getName() + "." + extension; //
+ } else {
+ // Assume file
+ //
+ FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(meta.getFilename()), space);
+ // originalPath = fileObject.getParent().getName().getPath();
+ // baseName = fileObject.getName().getBaseName();
+ fullname = fileObject.getName().getPath();
+ }
+ // } catch (FileSystemException e) {
+ // throw new KettleException(
+ // BaseMessages.getString(PKG, "JobMeta.Exception.AnErrorOccuredReadingJob", meta.getFilename()), e);
+ } catch (KettleFileException e) {
+ throw new KettleException(
+ BaseMessages.getString(PKG, "JobMeta.Exception.AnErrorOccuredReadingJob", meta.getFilename()), e);
+ }
+
+ // String resourceName = namingInterface.nameResource(
+ // baseName, originalPath, extension, ResourceNamingInterface.FileNamingType.JOB);
+ // logger.logBasic("=====> Checking [" + fullname + "] in " + definitions + " result=" + definitions.containsKey(fullname));
+ return definitions.containsKey(fullname) || meta.equals(space);
+ }
+
+ public static TransMeta loadTransformation(
+ Repository rep, RepositoryDirectoryInterface dir, String realFileName) throws KettleException {
+ TransMeta transMeta = null;
+ if (rep == null || dir == null || realFileName == null) {
+ return transMeta;
+ }
+
+ if (containsVariable(realFileName)) {
+ TransMetaCollection tmc = new TransMetaCollection();
+ transMeta = tmc;
+ transMeta.setFilename(realFileName);
+ for (RepositoryElementMetaInterface element : dir.getRepositoryObjects()) {
+ if (element.getObjectType() != RepositoryObjectType.TRANSFORMATION) {
+ continue;
+ }
+
+ tmc.attachTransMeta(rep.loadTransformation(element.getName(), dir, null, true, null));
+ }
+ } else {
+ transMeta = rep.loadTransformation(realFileName, dir, null, true, null);
+ }
+
+ return transMeta;
+ }
+
+ public static JobMeta loadJob(
+ Repository rep, RepositoryDirectoryInterface dir, String realFileName) throws KettleException {
+ JobMeta jobMeta = null;
+ if (rep == null || dir == null || realFileName == null) {
+ return jobMeta;
+ }
+
+ if (containsVariable(realFileName)) {
+ JobMetaCollection jmc = new JobMetaCollection();
+ jobMeta = jmc;
+ jobMeta.setFilename(realFileName);
+ for (RepositoryElementMetaInterface element : dir.getRepositoryObjects()) {
+ if (element.getObjectType() != RepositoryObjectType.JOB) {
+ continue;
+ }
+
+ jmc.attachJobMeta(rep.loadJob(element.getName(), dir, null, null));
+ }
+ } else {
+ jobMeta = rep.loadJob(realFileName, dir, null, null);
+ }
+
+ return jobMeta;
+ }
+
+ private ResourceDefinitionHelper() {
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceUtil.java b/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceUtil.java
new file mode 100644
index 0000000..361392b
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/resource/ResourceUtil.java
@@ -0,0 +1,172 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.resource;
+
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.core.Const;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.repository.Repository;
+import org.pentaho.di.trans.TransMeta;
+import org.pentaho.metastore.api.IMetaStore;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+public class ResourceUtil {
+
+ private static Class PKG = ResourceUtil.class; // for i18n purposes, needed by Translator2!!
+
+ /**
+ * Serializes the referenced resource export interface (Job, Transformation, Mapping, Step, Job Entry, etc) to a ZIP
+ * file.
+ *
+ * @param zipFilename The ZIP file to put the content in
+ * @param resourceExportInterface the interface to serialize
+ * @param space the space to use for variable replacement
+ * @param repository the repository to load objects from (or null if not used)
+ * @param metaStore the metaStore to load from
+ * @return The full VFS filename reference to the serialized export interface XML file in the ZIP archive.
+ * @throws KettleException in case anything goes wrong during serialization
+ */
+ public static final TopLevelResource serializeResourceExportInterface(String zipFilename,
+ ResourceExportInterface resourceExportInterface, VariableSpace space, Repository repository,
+ IMetaStore metaStore) throws KettleException {
+ return serializeResourceExportInterface(
+ zipFilename, resourceExportInterface, space, repository, metaStore, null, null);
+ }
+
+ /**
+ * Serializes the referenced resource export interface (Job, Transformation, Mapping, Step, Job Entry, etc) to a ZIP
+ * file.
+ *
+ * @param zipFilename The ZIP file to put the content in
+ * @param resourceExportInterface the interface to serialize
+ * @param space the space to use for variable replacement
+ * @param repository the repository to load objects from (or null if not used)
+ * @param injectXML The XML to inject into the resulting ZIP archive (optional, can be null)
+ * @param injectFilename The name of the file for the XML to inject in the ZIP archive (optional, can be null)
+ * @return The full VFS filename reference to the serialized export interface XML file in the ZIP archive.
+ * @throws KettleException in case anything goes wrong during serialization
+ */
+ public static final TopLevelResource serializeResourceExportInterface(String zipFilename,
+ ResourceExportInterface resourceExportInterface, VariableSpace space, Repository repository,
+ IMetaStore metaStore, String injectXML, String injectFilename) throws KettleException {
+
+ ZipOutputStream out = null;
+
+ try {
+ Map definitions = new HashMap();
+
+ // In case we want to add an extra pay-load to the exported ZIP file...
+ //
+ if (injectXML != null) {
+ ResourceDefinition resourceDefinition = new ResourceDefinition(injectFilename, injectXML);
+ definitions.put(injectFilename, resourceDefinition);
+ }
+
+ ResourceNamingInterface namingInterface = new SequenceResourceNaming();
+
+ String topLevelResource =
+ resourceExportInterface.exportResources(space, definitions, namingInterface, repository, metaStore);
+
+ if (topLevelResource != null && !definitions.isEmpty()) {
+
+ // Create the ZIP file...
+ //
+ FileObject fileObject = KettleVFS.getFileObject(zipFilename, space);
+
+ // Store the XML in the definitions in a ZIP file...
+ //
+ out = new ZipOutputStream(KettleVFS.getOutputStream(fileObject, false));
+
+ for (String filename : definitions.keySet()) {
+ ResourceDefinition resourceDefinition = definitions.get(filename);
+
+ ZipEntry zipEntry = new ZipEntry(resourceDefinition.getFilename());
+
+ String comment =
+ BaseMessages.getString(
+ PKG, "ResourceUtil.SerializeResourceExportInterface.ZipEntryComment.OriginatingFile", filename,
+ Const.NVL(resourceDefinition.getOrigin(), "-"));
+ zipEntry.setComment(comment);
+ out.putNextEntry(zipEntry);
+
+ out.write(resourceDefinition.getContent().getBytes());
+ out.closeEntry();
+ }
+ String zipURL = fileObject.getName().toString();
+ return new TopLevelResource(topLevelResource, zipURL, "zip:" + zipURL + "!" + topLevelResource);
+ } else {
+ throw new KettleException(BaseMessages.getString(PKG, "ResourceUtil.Exception.NoResourcesFoundToExport"));
+ }
+ } catch (Exception e) {
+ throw new KettleException(BaseMessages.getString(
+ PKG, "ResourceUtil.Exception.ErrorSerializingExportInterface", resourceExportInterface.toString()), e);
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ throw new KettleException(BaseMessages.getString(
+ PKG, "ResourceUtil.Exception.ErrorClosingZipStream", zipFilename));
+ }
+ }
+ }
+ }
+
+ public static String getExplanation(String zipFilename, String launchFile,
+ ResourceExportInterface resourceExportInterface) {
+
+ String commandString = "";
+ if (Const.isWindows()) {
+ if (resourceExportInterface instanceof TransMeta) {
+ commandString += "Pan.bat /file:\"";
+ } else {
+ commandString += "Kitchen.bat /file:\"";
+ }
+ } else {
+ if (resourceExportInterface instanceof TransMeta) {
+ commandString += "sh pan.sh -file='";
+ } else {
+ commandString += "sh kitchen.sh -file='";
+ }
+ }
+ commandString += launchFile;
+ if (Const.isWindows()) {
+ commandString += "\"";
+ } else {
+ commandString += "'";
+ }
+
+ String message =
+ BaseMessages.getString(
+ PKG, "ResourceUtil.ExportResourcesExplanation", zipFilename, commandString, launchFile, Const.CR);
+ return message;
+ }
+}
diff --git a/pentaho-kettle/src/main/java/org/pentaho/di/trans/Trans.java b/pentaho-kettle/src/main/java/org/pentaho/di/trans/Trans.java
new file mode 100644
index 0000000..9600ac6
--- /dev/null
+++ b/pentaho-kettle/src/main/java/org/pentaho/di/trans/Trans.java
@@ -0,0 +1,5502 @@
+//CHECKSTYLE:FileLength:OFF
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.pentaho.di.trans;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.pentaho.di.cluster.SlaveServer;
+import org.pentaho.di.core.*;
+import org.pentaho.di.core.database.Database;
+import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.database.DatabaseTransactionListener;
+import org.pentaho.di.core.database.map.DatabaseConnectionMap;
+import org.pentaho.di.core.exception.*;
+import org.pentaho.di.core.extension.ExtensionPointHandler;
+import org.pentaho.di.core.extension.KettleExtensionPoint;
+import org.pentaho.di.core.logging.*;
+import org.pentaho.di.core.metrics.MetricsDuration;
+import org.pentaho.di.core.metrics.MetricsSnapshotInterface;
+import org.pentaho.di.core.metrics.MetricsUtil;
+import org.pentaho.di.core.parameters.DuplicateParamException;
+import org.pentaho.di.core.parameters.NamedParams;
+import org.pentaho.di.core.parameters.NamedParamsDefault;
+import org.pentaho.di.core.parameters.UnknownParamException;
+import org.pentaho.di.core.row.RowMetaInterface;
+import org.pentaho.di.core.row.ValueMeta;
+import org.pentaho.di.core.util.EnvUtil;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.variables.Variables;
+import org.pentaho.di.core.vfs.KettleVFS;
+import org.pentaho.di.core.xml.XMLHandler;
+import org.pentaho.di.i18n.BaseMessages;
+import org.pentaho.di.job.DelegationListener;
+import org.pentaho.di.job.Job;
+import org.pentaho.di.partition.PartitionSchema;
+import org.pentaho.di.repository.ObjectId;
+import org.pentaho.di.repository.ObjectRevision;
+import org.pentaho.di.repository.Repository;
+import org.pentaho.di.repository.RepositoryDirectoryInterface;
+import org.pentaho.di.resource.ResourceUtil;
+import org.pentaho.di.resource.TopLevelResource;
+import org.pentaho.di.trans.cluster.TransSplitter;
+import org.pentaho.di.trans.performance.StepPerformanceSnapShot;
+import org.pentaho.di.trans.step.*;
+import org.pentaho.di.trans.step.BaseStepData.StepExecutionStatus;
+import org.pentaho.di.trans.steps.mappinginput.MappingInput;
+import org.pentaho.di.trans.steps.mappingoutput.MappingOutput;
+import org.pentaho.di.www.*;
+import org.pentaho.metastore.api.IMetaStore;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class represents the information and operations associated with the concept of a Transformation. It loads,
+ * instantiates, initializes, runs, and monitors the execution of the transformation contained in the specified
+ * TransInfo object.
+ *
+ * @author Matt
+ * @since 07-04-2003
+ */
+public class Trans implements VariableSpace, NamedParams, HasLogChannelInterface, LoggingObjectInterface,
+ ExecutorInterface, ExtensionDataInterface {
+
+ /**
+ * The package name, used for internationalization of messages.
+ */
+ private static Class PKG = Trans.class; // for i18n purposes, needed by Translator2!!
+
+ /**
+ * The replay date format.
+ */
+ public static final String REPLAY_DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
+
+ /**
+ * The log channel interface.
+ */
+ protected LogChannelInterface log;
+
+ /**
+ * The log level.
+ */
+ protected LogLevel logLevel = LogLevel.BASIC;
+
+ /**
+ * The container object id.
+ */
+ protected String containerObjectId;
+
+ /**
+ * The log commit size.
+ */
+ protected int logCommitSize = 10;
+
+ /**
+ * The transformation metadata to execute.
+ */
+ protected TransMeta transMeta;
+
+ /**
+ * The repository we are referencing.
+ */
+ protected Repository repository;
+
+ /**
+ * The MetaStore to use
+ */
+ protected IMetaStore metaStore;
+
+ /**
+ * The job that's launching this transformation. This gives us access to the whole chain, including the parent
+ * variables, etc.
+ */
+ private Job parentJob;
+
+ /**
+ * The transformation that is executing this transformation in case of mappings.
+ */
+ private Trans parentTrans;
+
+ /**
+ * The parent logging object interface (this could be a transformation or a job).
+ */
+ private LoggingObjectInterface parent;
+
+ /**
+ * The name of the mapping step that executes this transformation in case this is a mapping.
+ */
+ private String mappingStepName;
+
+ /**
+ * Indicates that we want to monitor the running transformation in a GUI.
+ */
+ private boolean monitored;
+
+ /**
+ * Indicates that we are running in preview mode...
+ */
+ private boolean preview;
+
+ /**
+ * The date objects for logging information about the transformation such as start and end time, etc.
+ */
+ private Date startDate, endDate, currentDate, logDate, depDate;
+
+ /**
+ * The job start and end date.
+ */
+ private Date jobStartDate, jobEndDate;
+
+ /**
+ * The batch id.
+ */
+ private long batchId;
+
+ /**
+ * This is the batch ID that is passed from job to job to transformation, if nothing is passed, it's the
+ * transformation's batch id.
+ */
+ private long passedBatchId;
+
+ /**
+ * The variable bindings for the transformation.
+ */
+ private VariableSpace variables = new Variables();
+
+ /**
+ * A list of all the row sets.
+ */
+ private List rowsets;
+
+ /**
+ * A list of all the steps.
+ */
+ private List steps;
+
+ /**
+ * The class number.
+ */
+ public int class_nr;
+
+ /**
+ * The replayDate indicates that this transformation is a replay transformation for a transformation executed on
+ * replayDate. If replayDate is null, the transformation is not a replay.
+ */
+ private Date replayDate;
+
+ /**
+ * Constant indicating a dispatch type of 1-to-1.
+ */
+ public static final int TYPE_DISP_1_1 = 1;
+
+ /**
+ * Constant indicating a dispatch type of 1-to-N.
+ */
+ public static final int TYPE_DISP_1_N = 2;
+
+ /**
+ * Constant indicating a dispatch type of N-to-1.
+ */
+ public static final int TYPE_DISP_N_1 = 3;
+
+ /**
+ * Constant indicating a dispatch type of N-to-N.
+ */
+ public static final int TYPE_DISP_N_N = 4;
+
+ /**
+ * Constant indicating a dispatch type of N-to-M.
+ */
+ public static final int TYPE_DISP_N_M = 5;
+
+ /**
+ * Constant indicating a transformation status of Finished.
+ */
+ public static final String STRING_FINISHED = "Finished";
+
+ /**
+ * Constant indicating a transformation status of Finished (with errors).
+ */
+ public static final String STRING_FINISHED_WITH_ERRORS = "Finished (with errors)";
+
+ /**
+ * Constant indicating a transformation status of Running.
+ */
+ public static final String STRING_RUNNING = "Running";
+
+ /**
+ * Constant indicating a transformation status of Paused.
+ */
+ public static final String STRING_PAUSED = "Paused";
+
+ /**
+ * Constant indicating a transformation status of Preparing for execution.
+ */
+ public static final String STRING_PREPARING = "Preparing executing";
+
+ /**
+ * Constant indicating a transformation status of Initializing.
+ */
+ public static final String STRING_INITIALIZING = "Initializing";
+
+ /**
+ * Constant indicating a transformation status of Waiting.
+ */
+ public static final String STRING_WAITING = "Waiting";
+
+ /**
+ * Constant indicating a transformation status of Stopped.
+ */
+ public static final String STRING_STOPPED = "Stopped";
+
+ /**
+ * Constant indicating a transformation status of Halting.
+ */
+ public static final String STRING_HALTING = "Halting";
+
+ /**
+ * Constant specifying a filename containing XML to inject into a ZIP file created during resource export.
+ */
+ public static final String CONFIGURATION_IN_EXPORT_FILENAME = "__job_execution_configuration__.xml";
+
+ /**
+ * Whether safe mode is enabled.
+ */
+ private boolean safeModeEnabled;
+
+ /**
+ * The thread name.
+ */
+ @Deprecated
+ private String threadName;
+
+ /**
+ * The transaction ID
+ */
+ private String transactionId;
+
+ /**
+ * Whether the transformation is preparing for execution.
+ */
+ private volatile boolean preparing;
+
+ /**
+ * Whether the transformation is initializing.
+ */
+ private boolean initializing;
+
+ /**
+ * Whether the transformation is running.
+ */
+ private boolean running;
+
+ /**
+ * Whether the transformation is finished.
+ */
+ private final AtomicBoolean finished;
+
+ /**
+ * Whether the transformation is paused.
+ */
+ private AtomicBoolean paused;
+
+ /**
+ * Whether the transformation is stopped.
+ */
+ private AtomicBoolean stopped;
+
+ /**
+ * The number of errors that have occurred during execution of the transformation.
+ */
+ private AtomicInteger errors;
+
+ /**
+ * Whether the transformation is ready to start.
+ */
+ private boolean readyToStart;
+
+ /**
+ * Step performance snapshots.
+ */
+ private Map> stepPerformanceSnapShots;
+
+ /**
+ * The step performance snapshot timer.
+ */
+ private Timer stepPerformanceSnapShotTimer;
+
+ /**
+ * A list of listeners attached to the transformation.
+ */
+ private List transListeners;
+
+ /**
+ * A list of stop-event listeners attached to the transformation.
+ */
+ private List transStoppedListeners;
+
+ /**
+ * In case this transformation starts to delegate work to a local transformation or job
+ */
+ private List delegationListeners;
+
+ /**
+ * The number of finished steps.
+ */
+ private int nrOfFinishedSteps;
+
+ /**
+ * The number of active steps.
+ */
+ private int nrOfActiveSteps;
+
+ /**
+ * The named parameters.
+ */
+ private NamedParams namedParams = new NamedParamsDefault();
+
+ /**
+ * The socket repository.
+ */
+ private SocketRepository socketRepository;
+
+ /**
+ * The transformation log table database connection.
+ */
+ private Database transLogTableDatabaseConnection;
+
+ /**
+ * The step performance snapshot sequence number.
+ */
+ private AtomicInteger stepPerformanceSnapshotSeqNr;
+
+ /**
+ * The last written step performance sequence number.
+ */
+ private int lastWrittenStepPerformanceSequenceNr;
+
+ /**
+ * The last step performance snapshot sequence number added.
+ */
+ private int lastStepPerformanceSnapshotSeqNrAdded;
+
+ /**
+ * The active subtransformations.
+ */
+ private Map activeSubtransformations;
+
+ /**
+ * The active subjobs
+ */
+ private Map activeSubjobs;
+
+ /**
+ * The step performance snapshot size limit.
+ */
+ private int stepPerformanceSnapshotSizeLimit;
+
+ /**
+ * The servlet print writer.
+ */
+ private PrintWriter servletPrintWriter;
+
+ /**
+ * The trans finished blocking queue.
+ */
+ private ArrayBlockingQueue transFinishedBlockingQueue;
+
+ /**
+ * The name of the executing server
+ */
+ private String executingServer;
+
+ /**
+ * The name of the executing user
+ */
+ private String executingUser;
+
+ private Result previousResult;
+
+ protected List resultRows;
+
+ protected List resultFiles;
+
+ /**
+ * The command line arguments for the transformation.
+ */
+ protected String[] arguments;
+
+ /**
+ * A table of named counters.
+ */
+ protected Hashtable counters;
+
+ private HttpServletResponse servletResponse;
+
+ private HttpServletRequest servletRequest;
+
+ private Map