From 32f3ccdec63ff5b1eed26586ccf536b031b608d6 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 6 Aug 2015 07:33:08 -0700 Subject: [PATCH 01/27] Initial add --- .../yahoo/validatar/execution/pig/Sty.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/main/java/com/yahoo/validatar/execution/pig/Sty.java diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java new file mode 100644 index 0000000..094a060 --- /dev/null +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014-2015 Yahoo! Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.yahoo.validatar.execution.pig; + +import com.yahoo.validatar.execution.Engine; +import com.yahoo.validatar.common.Query; +import com.yahoo.validatar.common.Result; +import com.yahoo.validatar.common.TypeSystem; +import com.yahoo.validatar.common.TypedObject; + +import static java.util.Arrays.*; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; + +import org.apache.log4j.Logger; + +import java.io.IOException; + +public class Sty implements Engine { + protected final Logger log = Logger.getLogger(getClass()); + + /** Engine name. */ + public static final String ENGINE_NAME = "pig"; + + + private OptionParser parser = new OptionParser() { + { + acceptsAll(asList("pig-setting"), "Settings and their values. Ex: 'mapreduce.job.acl-view-job=*'") + .withRequiredArg() + .describedAs("Pig generic settings to use."); + allowsUnrecognizedOptions(); + } + }; + + /** {@inheritDoc} */ + @Override + public String getName() { + return ENGINE_NAME; + } + + /** {@inheritDoc} */ + @Override + public boolean setup(String[] arguments) { + OptionSet options = parser.parse(arguments); + return true; + } + + /** {@inheritDoc} */ + @Override + public void printHelp() { + System.out.println(ENGINE_NAME + " help:"); + try { + parser.printHelpOn(System.out); + } catch (IOException e) { + log.error(e); + } + System.out.println(); + } + + /** {@inheritDoc} */ + @Override + public void execute(Query query) { + Result result = query.createResults(); + } +} From d16700f13e56bdae7c71811408592db50d47adf2 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 11 Sep 2015 01:50:03 -0700 Subject: [PATCH 02/27] Basic Pig framework. Need to fetch results --- pom.xml | 17 +++- .../validatar/execution/hive/Apiary.java | 4 +- .../yahoo/validatar/execution/pig/Sty.java | 95 ++++++++++++++++--- 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/pom.xml b/pom.xml index 3458a3e..21a3402 100644 --- a/pom.xml +++ b/pom.xml @@ -199,6 +199,12 @@ 1.9.5 test + + com.h2database + h2 + 1.4.180 + test + org.antlr antlr4-runtime @@ -229,16 +235,17 @@ commons-lang3 3.3.2 - - com.h2database - h2 - 1.4.180 - org.reflections reflections 0.9.9-RC1 + + org.apache.pig + pig + 0.14.0 + provided + diff --git a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java index 8d46eb9..733659d 100644 --- a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java +++ b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java @@ -99,7 +99,7 @@ public void printHelp() { public void execute(Query query) { String queryName = query.name; String queryValue = query.value; - log.info("Running: " + queryValue); + log.info("Running " + queryName + ": " + queryValue); try { ResultSet result = statement.executeQuery(queryValue); ResultSetMetaData metadata = result.getMetaData(); @@ -113,7 +113,7 @@ public void execute(Query query) { } result.close(); } catch (SQLException e) { - log.error("SQL problem with query: " + queryName + "\n" + queryValue, e); + log.error("SQL problem with Hive query: " + queryName + "\n" + queryValue, e); query.setFailure(e.toString()); } } diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 094a060..490eac0 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -16,20 +16,27 @@ package com.yahoo.validatar.execution.pig; -import com.yahoo.validatar.execution.Engine; +import com.yahoo.validatar.common.Metadata; import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.Result; -import com.yahoo.validatar.common.TypeSystem; -import com.yahoo.validatar.common.TypedObject; - -import static java.util.Arrays.*; - +import com.yahoo.validatar.execution.Engine; import joptsimple.OptionParser; import joptsimple.OptionSet; - import org.apache.log4j.Logger; +import org.apache.pig.PigServer; +import org.apache.pig.data.DataType; +import org.apache.pig.data.Tuple; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; public class Sty implements Engine { protected final Logger log = Logger.getLogger(getClass()); @@ -37,30 +44,52 @@ public class Sty implements Engine { /** Engine name. */ public static final String ENGINE_NAME = "pig"; + public static final String DEFAULT_EXEC_TYPE = "mr"; + public static final String DEFAULT_OUTPUT_ALIAS = "validatar_results"; + public static final String SETTING_DELIMITER = "="; + + public static final String METADATA_EXEC_TYPE_KEY = "exec-type"; + public static final String METADATA_ALIAS_KEY = "output-alias"; + + private String defaultExecType; + private String defaultOutputAlias; + private Properties properties; private OptionParser parser = new OptionParser() { { - acceptsAll(asList("pig-setting"), "Settings and their values. Ex: 'mapreduce.job.acl-view-job=*'") - .withRequiredArg() - .describedAs("Pig generic settings to use."); + acceptsAll(singletonList("pig-exec-type"), "The exec-type for Pig to use. This is the " + + "-x argument used when running Pig. Ex: local, mr, tez etc. ") + .withRequiredArg() + .describedAs("Hive JDBC connector") + .defaultsTo(DEFAULT_EXEC_TYPE); + acceptsAll(singletonList("pig-output-alias"), "The name of the alias where the result of the Pig script is." + + "This should contain the data that will be collected") + .withRequiredArg() + .describedAs("Pig default output alias") + .defaultsTo(DEFAULT_OUTPUT_ALIAS); + acceptsAll(asList("pig-setting"), "Settings and their values. The -D params that would " + + "have been sent to Pig. Ex: 'mapreduce.job.acl-view-job=*'") + .withRequiredArg() + .describedAs("Pig generic settings to use."); allowsUnrecognizedOptions(); } }; - /** {@inheritDoc} */ @Override public String getName() { return ENGINE_NAME; } - /** {@inheritDoc} */ @Override public boolean setup(String[] arguments) { OptionSet options = parser.parse(arguments); + defaultExecType = (String) options.valueOf("pig-exec-type"); + defaultOutputAlias = (String) options.valueOf("pig-output-alias"); + properties = getProperties(options); + // We will boot up a PigServer per query, so nothing else to do... return true; } - /** {@inheritDoc} */ @Override public void printHelp() { System.out.println(ENGINE_NAME + " help:"); @@ -72,9 +101,45 @@ public void printHelp() { System.out.println(); } - /** {@inheritDoc} */ @Override public void execute(Query query) { - Result result = query.createResults(); + String queryName = query.name; + String queryValue = query.value; + Map queryMetadata = query.getMetadata(); + String execType = getKey(queryMetadata, METADATA_EXEC_TYPE_KEY).orElse(defaultExecType); + String alias = getKey(queryMetadata, METADATA_ALIAS_KEY).orElse(defaultOutputAlias); + log.info("Running " + queryName + " for alias " + alias + ": " + queryValue); + try { + PigServer server = new PigServer(execType, properties); + server.registerScript(new ByteArrayInputStream(queryValue.getBytes())); + Iterator queryResults = server.openIterator(alias); + Result result = query.createResults(); + server.shutdown(); + } catch (IOException e) { + log.error("Problem with Pig query: " + queryName + "\n" + queryValue, e); + query.setFailure(e.toString()); + } catch (Exception e) { + log.error("Unexpected error occurred while executing Pig query: " + queryName + "\n" + queryValue, e); + } + } + + private Optional getKey(Map metadata, String key) { + String value = metadata.get(key); + return value == null || value.isEmpty() ? Optional.empty(): Optional.of(value); } + + private Properties getProperties(OptionSet options) { + List settings = (List) options.valuesOf("pig-setting"); + Properties properties = new Properties(); + for (String setting : settings) { + String[] tokens = setting.split(SETTING_DELIMITER); + if (tokens.length != 2) { + log.error("Ignoring unknown Pig setting provided: " + setting); + continue; + } + properties.put(tokens[0], tokens[1]); + } + return properties; + } + } From 000e66fe6e2fdec8db2963bd6c48ec166c5a9518 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 11 Sep 2015 10:02:57 -0700 Subject: [PATCH 03/27] Pig overall. Untested. Modified Hive to use helpers. --- .../validatar/execution/hive/Apiary.java | 12 +-- .../yahoo/validatar/execution/pig/Sty.java | 96 ++++++++++++++++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java index 733659d..d9f321d 100644 --- a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java +++ b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java @@ -156,26 +156,26 @@ TypedObject getAsTypedObject(ResultSet results, int index, int type) throws SQLE case (Types.DATE): case (Types.CHAR): case (Types.VARCHAR): - toReturn = new TypedObject(results.getString(index), TypeSystem.Type.STRING); + toReturn = TypeSystem.asTypedObject(results.getString(index)); break; case (Types.FLOAT): case (Types.DOUBLE): - toReturn = new TypedObject(results.getDouble(index), TypeSystem.Type.DOUBLE); + toReturn = TypeSystem.asTypedObject(results.getDouble(index)); break; case (Types.BOOLEAN): - toReturn = new TypedObject(results.getBoolean(index), TypeSystem.Type.BOOLEAN); + toReturn = TypeSystem.asTypedObject(results.getBoolean(index)); break; case (Types.TINYINT): case (Types.SMALLINT): case (Types.INTEGER): case (Types.BIGINT): - toReturn = new TypedObject(results.getLong(index), TypeSystem.Type.LONG); + toReturn = TypeSystem.asTypedObject(results.getLong(index)); break; case (Types.DECIMAL): - toReturn = new TypedObject(results.getBigDecimal(index), TypeSystem.Type.DECIMAL); + toReturn = TypeSystem.asTypedObject(results.getBigDecimal(index)); break; case (Types.TIMESTAMP): - toReturn = new TypedObject(results.getTimestamp(index), TypeSystem.Type.TIMESTAMP); + toReturn = TypeSystem.asTypedObject(results.getTimestamp(index)); break; default: throw new UnsupportedOperationException("Unknown SQL type encountered from Hive: " + type); diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 490eac0..eb6641b 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -16,19 +16,25 @@ package com.yahoo.validatar.execution.pig; -import com.yahoo.validatar.common.Metadata; import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.Result; +import com.yahoo.validatar.common.TypeSystem; +import com.yahoo.validatar.common.TypedObject; import com.yahoo.validatar.execution.Engine; import joptsimple.OptionParser; import joptsimple.OptionSet; import org.apache.log4j.Logger; import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.apache.tools.ant.taskdefs.Exec; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -55,6 +61,16 @@ public class Sty implements Engine { private String defaultOutputAlias; private Properties properties; + private class FieldDetail { + public String alias; + public byte type; + + public FieldDetail(String alias, byte type) { + this.alias = alias; + this.type = type; + } + } + private OptionParser parser = new OptionParser() { { acceptsAll(singletonList("pig-exec-type"), "The exec-type for Pig to use. This is the " + @@ -113,14 +129,84 @@ public void execute(Query query) { PigServer server = new PigServer(execType, properties); server.registerScript(new ByteArrayInputStream(queryValue.getBytes())); Iterator queryResults = server.openIterator(alias); + Result result = query.createResults(); + List metadata = getFieldDetails(server.dumpSchema(alias)); + populateColumns(metadata, result); + while (queryResults.hasNext()) { + populateRow(queryResults.next(), metadata, result); + } server.shutdown(); - } catch (IOException e) { - log.error("Problem with Pig query: " + queryName + "\n" + queryValue, e); - query.setFailure(e.toString()); + } catch (IOException ioe) { + log.error("Problem with Pig query: \n" + queryValue, ioe); + query.setFailure(ioe.toString()); } catch (Exception e) { - log.error("Unexpected error occurred while executing Pig query: " + queryName + "\n" + queryValue, e); + log.error("Error occurred while processing Pig query: " + queryValue, e); + query.setFailure(e.toString()); + } + } + + private void populateColumns(List metadata, Result result) throws IOException { + if (metadata == null || metadata.isEmpty()) { + throw new IOException("No metaadata of columns found for Pig query"); + } + metadata.forEach(m -> result.addColumn(m.alias)); + } + + private void populateRow(Tuple row, List metadata, Result result) throws ExecException { + if (row == null) { + return; + } + for (int i = 0; i < metadata.size(); ++i) { + FieldDetail column = metadata.get(i); + TypedObject value = getTypedObject(row.get(i), column); + log.info("Column: " + column.alias + "\tType: " + column.type + + "\tValue: " + (value == null ? "null" : value.data)); + result.addColumnRow(column.alias, value); + } + } + + private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { + byte type = detail.type; + switch (type) { + case DataType.BOOLEAN: + return TypeSystem.asTypedObject(DataType.toBoolean(data, type)); + case DataType.INTEGER: + case DataType.LONG: + return TypeSystem.asTypedObject(DataType.toLong(data, type)); + case DataType.FLOAT: + case DataType.DOUBLE: + case DataType.DATETIME: + return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis())); + case DataType.BYTE: + case DataType.BYTEARRAY: + case DataType.CHARARRAY: + return TypeSystem.asTypedObject(DataType.toString(data, type)); + case DataType.BIGINTEGER: + case DataType.BIGDECIMAL: + return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type)); + case DataType.TUPLE: + case DataType.BAG: + case DataType.MAP: + case DataType.INTERNALMAP: + case DataType.GENERIC_WRITABLECOMPARABLE: + case DataType.ERROR: + case DataType.UNKNOWN: + case DataType.NULL: + default: + return null; + } + } + + private List getFieldDetails(Schema schema) { + List details = new ArrayList<>(); + if (schema == null) { + return details; + } + for (Schema.FieldSchema field : schema.getFields()) { + details.add(new FieldDetail(field.alias, field.type)); } + return details; } private Optional getKey(Map metadata, String key) { From a90c428fcf1b59628158de1531ef1679b5c9fffc Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Sat, 12 Sep 2015 10:16:42 -0700 Subject: [PATCH 04/27] Extracting option constants. Adding hadoop. Cleaning up Pig. --- pom.xml | 15 ++++--- src/main/java/com/yahoo/validatar/App.java | 23 ++++++----- .../validatar/execution/hive/Apiary.java | 32 ++++++++------- .../yahoo/validatar/execution/pig/Sty.java | 40 +++++++++---------- .../yahoo/validatar/report/FormatManager.java | 9 ++--- .../report/junit/JUnitFormatter.java | 5 ++- .../validatar/execution/hive/ApiaryTest.java | 2 +- .../report/junit/JUnitFormatterTest.java | 2 +- 8 files changed, 66 insertions(+), 62 deletions(-) diff --git a/pom.xml b/pom.xml index 21a3402..dc45c8b 100644 --- a/pom.xml +++ b/pom.xml @@ -156,14 +156,6 @@ org.codehaus.mojo cobertura-maven-plugin 2.7 - - - - org.ow2.asm - asm - 5.0.3 - - html @@ -174,6 +166,7 @@ com/yahoo/validatar/assertion/Grammar*.class + @@ -240,6 +233,12 @@ reflections 0.9.9-RC1 + + org.apache.hadoop + hadoop-common + 2.6.0 + provided + org.apache.pig pig diff --git a/src/main/java/com/yahoo/validatar/App.java b/src/main/java/com/yahoo/validatar/App.java index 6e43d65..16568a3 100644 --- a/src/main/java/com/yahoo/validatar/App.java +++ b/src/main/java/com/yahoo/validatar/App.java @@ -42,25 +42,28 @@ import static java.util.Collections.singletonList; public class App { - /** - * Logging class. - */ protected static final Logger LOG = Logger.getLogger(com.yahoo.validatar.App.class.getName()); + public static final String PARAMETER = "parameter"; + public static final String PARAMETER_DELIMITER = "="; + public static final String TEST_SUITE = "test-suite"; + public static final String HELP = "help"; + public static final String HELP_ABBREVIATED = "h"; + /** * The CLI parser. */ public static final OptionParser PARSER = new OptionParser() { { - acceptsAll(singletonList("parameter"), "Parameter to replace all '${VAR}' in the query string. Ex: --parameter DATE=2014-07-24") + acceptsAll(singletonList(PARAMETER), "Parameter to replace all '${VAR}' in the query string. Ex: --parameter DATE=2014-07-24") .withRequiredArg() .describedAs("Parameter"); - acceptsAll(singletonList("test-suite"), "File or folder that contains the test suite file(s).") + acceptsAll(singletonList(TEST_SUITE), "File or folder that contains the test suite file(s).") .withRequiredArg() .required() .ofType(File.class) .describedAs("Test suite file/folder"); - acceptsAll(asList("h", "help"), "Shows help message."); + acceptsAll(asList(HELP_ABBREVIATED, HELP), "Shows help message."); allowsUnrecognizedOptions(); } }; @@ -75,7 +78,7 @@ public class App { public static Map splitParameters(OptionSet options, String parameterName) { Map parameterMap = new HashMap<>(); for (String parameter : (List) options.valuesOf(parameterName)) { - String[] tokens = parameter.split("="); + String[] tokens = parameter.split(PARAMETER_DELIMITER); if (tokens.length != 2) { throw new RuntimeException("Invalid parameter. It should be KEY=VALUE. Found " + parameter); } @@ -161,13 +164,13 @@ public static void main(String[] args) throws IOException { FormatManager formatManager = new FormatManager(args); // Check if user needs help - if (options == null || options.has("h") || options.has("help")) { + if (options == null || options.has(HELP_ABBREVIATED) || options.has(HELP)) { Helpable.printHelp("Application options", PARSER); engineManager.printHelp(); formatManager.printHelp(); return; } - Map parameterMap = splitParameters(options, "parameter"); - run((File) options.valueOf("test-suite"), parameterMap, parseManager, engineManager, formatManager); + Map parameterMap = splitParameters(options, PARAMETER); + run((File) options.valueOf(TEST_SUITE), parameterMap, parseManager, engineManager, formatManager); } } diff --git a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java index d9f321d..8ee4344 100644 --- a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java +++ b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java @@ -38,11 +38,13 @@ import static java.util.Collections.singletonList; public class Apiary implements Engine { + public static final String HIVE_JDBC = "hive-jdbc"; + public static final String HIVE_DRIVER = "hive-driver"; + public static final String HIVE_USERNAME = "hive-username"; + public static final String HIVE_PASSWORD = "hive-password"; + public static final String HIVE_SETTING = "hive-setting"; protected final Logger log = Logger.getLogger(getClass()); - /** - * Engine name. - */ public static final String ENGINE_NAME = "hive"; public static String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver"; @@ -52,25 +54,25 @@ public class Apiary implements Engine { private OptionParser parser = new OptionParser() { { - acceptsAll(singletonList("hive-jdbc"), "JDBC string to the HiveServer2 with an optional database. " + - "If the database is provided, the queries must NOT have one. " + - "Ex: 'jdbc:hive2://HIVE_SERVER:PORT/[DATABASE_FOR_ALL_QUERIES]' ") + acceptsAll(singletonList(HIVE_JDBC), "JDBC string to the HiveServer2 with an optional database. " + + "If the database is provided, the queries must NOT have one. " + + "Ex: 'jdbc:hive2://HIVE_SERVER:PORT/[DATABASE_FOR_ALL_QUERIES]' ") .withRequiredArg() .required() .describedAs("Hive JDBC connector"); - acceptsAll(singletonList("hive-driver"), "Fully qualified package name to the hive driver.") + acceptsAll(singletonList(HIVE_DRIVER), "Fully qualified package name to the hive driver.") .withRequiredArg() .describedAs("Hive driver") .defaultsTo(DRIVER_NAME); - acceptsAll(singletonList("hive-username"), "Hive server username.") + acceptsAll(singletonList(HIVE_USERNAME), "Hive server username.") .withRequiredArg() .describedAs("Hive server username") .defaultsTo("anon"); - acceptsAll(singletonList("hive-password"), "Hive server password.") + acceptsAll(singletonList(HIVE_PASSWORD), "Hive server password.") .withRequiredArg() .describedAs("Hive server password") .defaultsTo("anon"); - acceptsAll(singletonList("hive-setting"), "Settings and their values. Ex: 'hive.execution.engine=mr'") + acceptsAll(singletonList(HIVE_SETTING), "Settings and their values. Ex: 'hive.execution.engine=mr'") .withRequiredArg() .describedAs("Hive generic settings to use."); allowsUnrecognizedOptions(); @@ -193,16 +195,16 @@ TypedObject getAsTypedObject(ResultSet results, int index, int type) throws SQLE */ Statement setupConnection(OptionSet options) throws ClassNotFoundException, SQLException { // Load the JDBC driver - String driver = (String) options.valueOf("hive-driver"); + String driver = (String) options.valueOf(HIVE_DRIVER); log.info("Loading JDBC driver: " + driver); Class.forName(driver); // Get the JDBC connector - String jdbcConnector = (String) options.valueOf("hive-jdbc"); + String jdbcConnector = (String) options.valueOf(HIVE_JDBC); log.info("Connecting to: " + jdbcConnector); - String username = (String) options.valueOf("hive-username"); - String password = (String) options.valueOf("hive-password"); + String username = (String) options.valueOf(HIVE_USERNAME); + String password = (String) options.valueOf(HIVE_PASSWORD); // Start the connection Connection connection = DriverManager.getConnection(jdbcConnector, username, password); @@ -217,7 +219,7 @@ Statement setupConnection(OptionSet options) throws ClassNotFoundException, SQLE * @throws java.sql.SQLException if any. */ void setHiveSettings(OptionSet options, Statement statement) throws SQLException { - for (String setting : (List) options.valuesOf("hive-setting")) { + for (String setting : (List) options.valuesOf(HIVE_SETTING)) { log.info("Applying setting " + setting); statement.executeUpdate(SETTING_PREFIX + setting); } diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index eb6641b..b63d40e 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -29,22 +29,24 @@ import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; -import org.apache.tools.ant.taskdefs.Exec; import java.io.ByteArrayInputStream; import java.io.IOException; import java.sql.Timestamp; -import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.stream.Collectors; -import static java.util.Arrays.asList; import static java.util.Collections.singletonList; public class Sty implements Engine { + public static final String PIG_EXEC_TYPE = "pig-exec-type"; + public static final String PIG_OUTPUT_ALIAS = "pig-output-alias"; + public static final String PIG_SETTING = "pig-setting"; protected final Logger log = Logger.getLogger(getClass()); /** Engine name. */ @@ -73,18 +75,18 @@ public FieldDetail(String alias, byte type) { private OptionParser parser = new OptionParser() { { - acceptsAll(singletonList("pig-exec-type"), "The exec-type for Pig to use. This is the " + - "-x argument used when running Pig. Ex: local, mr, tez etc. ") + acceptsAll(singletonList(PIG_EXEC_TYPE), "The exec-type for Pig to use. This is the " + + "-x argument used when running Pig. Ex: local, mr, tez etc. ") .withRequiredArg() - .describedAs("Hive JDBC connector") + .describedAs("Pig execution type") .defaultsTo(DEFAULT_EXEC_TYPE); - acceptsAll(singletonList("pig-output-alias"), "The name of the alias where the result of the Pig script is." + - "This should contain the data that will be collected") + acceptsAll(singletonList(PIG_OUTPUT_ALIAS), "The default name of the alias where the result is." + + "This should contain the data that will be collected") .withRequiredArg() .describedAs("Pig default output alias") .defaultsTo(DEFAULT_OUTPUT_ALIAS); - acceptsAll(asList("pig-setting"), "Settings and their values. The -D params that would " + - "have been sent to Pig. Ex: 'mapreduce.job.acl-view-job=*'") + acceptsAll(singletonList(PIG_SETTING), "Settings and their values. The -D params that would " + + "have been sent to Pig. Ex: 'mapreduce.job.acl-view-job=*'") .withRequiredArg() .describedAs("Pig generic settings to use."); allowsUnrecognizedOptions(); @@ -99,8 +101,8 @@ public String getName() { @Override public boolean setup(String[] arguments) { OptionSet options = parser.parse(arguments); - defaultExecType = (String) options.valueOf("pig-exec-type"); - defaultOutputAlias = (String) options.valueOf("pig-output-alias"); + defaultExecType = (String) options.valueOf(PIG_EXEC_TYPE); + defaultOutputAlias = (String) options.valueOf(PIG_OUTPUT_ALIAS); properties = getProperties(options); // We will boot up a PigServer per query, so nothing else to do... return true; @@ -147,7 +149,7 @@ public void execute(Query query) { } private void populateColumns(List metadata, Result result) throws IOException { - if (metadata == null || metadata.isEmpty()) { + if (metadata.isEmpty()) { throw new IOException("No metaadata of columns found for Pig query"); } metadata.forEach(m -> result.addColumn(m.alias)); @@ -155,6 +157,7 @@ private void populateColumns(List metadata, Result result) throws I private void populateRow(Tuple row, List metadata, Result result) throws ExecException { if (row == null) { + log.info("Skipping null row in results..."); return; } for (int i = 0; i < metadata.size(); ++i) { @@ -176,6 +179,7 @@ private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecE return TypeSystem.asTypedObject(DataType.toLong(data, type)); case DataType.FLOAT: case DataType.DOUBLE: + return TypeSystem.asTypedObject(DataType.toDouble(data, type)); case DataType.DATETIME: return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis())); case DataType.BYTE: @@ -199,14 +203,10 @@ private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecE } private List getFieldDetails(Schema schema) { - List details = new ArrayList<>(); if (schema == null) { - return details; + return Collections.EMPTY_LIST; } - for (Schema.FieldSchema field : schema.getFields()) { - details.add(new FieldDetail(field.alias, field.type)); - } - return details; + return schema.getFields().stream().map(f -> new FieldDetail(f.alias, f.type)).collect(Collectors.toList()); } private Optional getKey(Map metadata, String key) { @@ -215,7 +215,7 @@ private Optional getKey(Map metadata, String key) { } private Properties getProperties(OptionSet options) { - List settings = (List) options.valuesOf("pig-setting"); + List settings = (List) options.valuesOf(PIG_SETTING); Properties properties = new Properties(); for (String setting : settings) { String[] tokens = setting.split(SETTING_DELIMITER); diff --git a/src/main/java/com/yahoo/validatar/report/FormatManager.java b/src/main/java/com/yahoo/validatar/report/FormatManager.java index 9c6a880..e6f6fbd 100644 --- a/src/main/java/com/yahoo/validatar/report/FormatManager.java +++ b/src/main/java/com/yahoo/validatar/report/FormatManager.java @@ -34,9 +34,8 @@ * Manages the writing of test reports. */ public class FormatManager implements Helpable { - /** - * Used for logging. - */ + public static final String REPORT_FORMAT = "report-format"; + protected Logger log = Logger.getLogger(getClass().getName()); private Map availableFormatters; @@ -46,7 +45,7 @@ public class FormatManager implements Helpable { // it can be moved to inside the respective formatters. private OptionParser parser = new OptionParser() { { - acceptsAll(singletonList("report-format"), "Which report format to use.") + acceptsAll(singletonList(REPORT_FORMAT), "Which report format to use.") .withRequiredArg() .describedAs("Report format") .defaultsTo("junit"); @@ -60,7 +59,7 @@ public class FormatManager implements Helpable { * @param arguments An array of parameters of the form [--param1 value1 --param2 value2...] */ public FormatManager(String[] arguments) { - String name = (String) parser.parse(arguments).valueOf("report-format"); + String name = (String) parser.parse(arguments).valueOf(REPORT_FORMAT); formatterToUse = findFormatter(name, arguments); if (formatterToUse == null) { printHelp(); diff --git a/src/main/java/com/yahoo/validatar/report/junit/JUnitFormatter.java b/src/main/java/com/yahoo/validatar/report/junit/JUnitFormatter.java index cf2be7f..8debe7c 100644 --- a/src/main/java/com/yahoo/validatar/report/junit/JUnitFormatter.java +++ b/src/main/java/com/yahoo/validatar/report/junit/JUnitFormatter.java @@ -37,13 +37,14 @@ import static java.util.Collections.singletonList; public class JUnitFormatter implements Formatter { + public static final String REPORT_FILE = "report-file"; protected final Logger log = Logger.getLogger(getClass()); public static final String JUNIT = "junit"; private OptionParser parser = new OptionParser() { { - acceptsAll(singletonList("report-file"), "File to store the test reports.") + acceptsAll(singletonList(REPORT_FILE), "File to store the test reports.") .withRequiredArg() .describedAs("Report file") .defaultsTo("report.xml"); @@ -54,7 +55,7 @@ public class JUnitFormatter implements Formatter { @Override public boolean setup(String[] arguments) { - outputFile = (String) parser.parse(arguments).valueOf("report-file"); + outputFile = (String) parser.parse(arguments).valueOf(REPORT_FILE); return true; } diff --git a/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java b/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java index 6b4e7a6..9d7dec4 100644 --- a/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java +++ b/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java @@ -49,7 +49,7 @@ public class ApiaryTest { "--hive-setting", "mapreduce.job.queuename=default"}; @Test - public void testGetJDBCConnector() throws ClassNotFoundException, SQLException, Exception { + public void testGetJDBCConnector() throws Exception { Apiary apiary = spy(new Apiary()); doNothing().when(apiary).setHiveSettings(any(OptionSet.class), any(Statement.class)); Assert.assertTrue(apiary.setup(args)); diff --git a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java index 3239ed7..7785896 100644 --- a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java +++ b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java @@ -35,7 +35,7 @@ public class JUnitFormatterTest { @Test - public void testWriteReport() throws FileNotFoundException, IOException, org.dom4j.DocumentException { + public void testWriteReport() throws IOException, org.dom4j.DocumentException { Map paramMap = new HashMap<>(); paramMap.put("DATE", "20140807"); From 3bd920d7aa0866b698312ddf1bad76fa10918eaa Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Sat, 12 Sep 2015 10:20:45 -0700 Subject: [PATCH 05/27] Fixing checkstyle and adding other Pig dependencies --- pom.xml | 12 ++++++++++++ .../java/com/yahoo/validatar/execution/pig/Sty.java | 2 +- .../validatar/report/junit/JUnitFormatterTest.java | 1 - 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index dc45c8b..ce53576 100644 --- a/pom.xml +++ b/pom.xml @@ -239,6 +239,18 @@ 2.6.0 provided + + org.apache.hadoop + hadoop-client + 2.6.0 + provided + + + joda-time + joda-time + 2.6 + provided + org.apache.pig pig diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index b63d40e..3e678dd 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -211,7 +211,7 @@ private List getFieldDetails(Schema schema) { private Optional getKey(Map metadata, String key) { String value = metadata.get(key); - return value == null || value.isEmpty() ? Optional.empty(): Optional.of(value); + return value == null || value.isEmpty() ? Optional.empty() : Optional.of(value); } private Properties getProperties(OptionSet options) { diff --git a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java index 7785896..917884d 100644 --- a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java +++ b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java @@ -25,7 +25,6 @@ import org.testng.annotations.Test; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; From 633c461bf4f50f23c21184b980b9266a676d37e7 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Sat, 12 Sep 2015 14:32:48 -0700 Subject: [PATCH 06/27] Adding a full pig test. Cleaning up some more. Cobertura is broken --- pom.xml | 8 ++++ .../yahoo/validatar/execution/pig/Sty.java | 16 +++---- src/main/resources/log4j.properties | 2 +- .../{LogCaptor.java => OutputCaptor.java} | 18 +++++++- .../execution/EngineManagerTest.java | 4 +- .../validatar/execution/pig/StyTest.java | 46 +++++++++++++++++++ src/test/resources/log4j.properties | 2 +- src/test/resources/pig-tests/data.txt | 9 ++++ src/test/resources/pig-tests/sample.yaml | 23 ++++++++++ 9 files changed, 113 insertions(+), 15 deletions(-) rename src/test/java/com/yahoo/validatar/{LogCaptor.java => OutputCaptor.java} (83%) create mode 100644 src/test/java/com/yahoo/validatar/execution/pig/StyTest.java create mode 100644 src/test/resources/pig-tests/data.txt create mode 100644 src/test/resources/pig-tests/sample.yaml diff --git a/pom.xml b/pom.xml index ce53576..31d893c 100644 --- a/pom.xml +++ b/pom.xml @@ -198,6 +198,12 @@ 1.4.180 test + + commons-io + commons-io + 2.4 + test + org.antlr antlr4-runtime @@ -213,6 +219,7 @@ log4j 1.2.17 + org.yaml snakeyaml @@ -255,6 +262,7 @@ org.apache.pig pig 0.14.0 + h2 provided diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 3e678dd..5c854d6 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -16,6 +16,7 @@ package com.yahoo.validatar.execution.pig; +import com.yahoo.validatar.common.Helpable; import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.Result; import com.yahoo.validatar.common.TypeSystem; @@ -47,6 +48,7 @@ public class Sty implements Engine { public static final String PIG_EXEC_TYPE = "pig-exec-type"; public static final String PIG_OUTPUT_ALIAS = "pig-output-alias"; public static final String PIG_SETTING = "pig-setting"; + protected final Logger log = Logger.getLogger(getClass()); /** Engine name. */ @@ -110,13 +112,7 @@ public boolean setup(String[] arguments) { @Override public void printHelp() { - System.out.println(ENGINE_NAME + " help:"); - try { - parser.printHelpOn(System.out); - } catch (IOException e) { - log.error(e); - } - System.out.println(); + Helpable.printHelp("Pig engine options", parser); } @Override @@ -131,8 +127,8 @@ public void execute(Query query) { PigServer server = new PigServer(execType, properties); server.registerScript(new ByteArrayInputStream(queryValue.getBytes())); Iterator queryResults = server.openIterator(alias); - Result result = query.createResults(); + // dumpSchema will also, unfortunately, print the schema to stdout. List metadata = getFieldDetails(server.dumpSchema(alias)); populateColumns(metadata, result); while (queryResults.hasNext()) { @@ -150,7 +146,7 @@ public void execute(Query query) { private void populateColumns(List metadata, Result result) throws IOException { if (metadata.isEmpty()) { - throw new IOException("No metaadata of columns found for Pig query"); + throw new IOException("No metadata of columns found for Pig query"); } metadata.forEach(m -> result.addColumn(m.alias)); } @@ -204,7 +200,7 @@ private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecE private List getFieldDetails(Schema schema) { if (schema == null) { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } return schema.getFields().stream().map(f -> new FieldDetail(f.alias, f.type)).collect(Collectors.toList()); } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 36ed542..2ac8315 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ # Root logger option -log4j.rootLogger= DEBUG, stdout +log4j.rootLogger= INFO, stdout # Direct log messages to stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/src/test/java/com/yahoo/validatar/LogCaptor.java b/src/test/java/com/yahoo/validatar/OutputCaptor.java similarity index 83% rename from src/test/java/com/yahoo/validatar/LogCaptor.java rename to src/test/java/com/yahoo/validatar/OutputCaptor.java index a8fbc83..ce6fe94 100644 --- a/src/test/java/com/yahoo/validatar/LogCaptor.java +++ b/src/test/java/com/yahoo/validatar/OutputCaptor.java @@ -16,12 +16,16 @@ package com.yahoo.validatar; +import org.apache.commons.io.output.NullOutputStream; import org.apache.log4j.Appender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; import org.mockito.ArgumentCaptor; +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.io.PrintStream; import java.util.List; import static org.mockito.Mockito.atLeastOnce; @@ -35,11 +39,15 @@ * the capturing and use isStringInLog to check if desired output in log * was seen. */ -public class LogCaptor { +public class OutputCaptor { protected Appender mockedAppender; protected Level originalLevel; protected Appender originalAppender; + public static final PrintStream NULL = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM); + public static final PrintStream OUT = new PrintStream(new FileOutputStream(FileDescriptor.out)); + public static final PrintStream ERR = new PrintStream(new FileOutputStream(FileDescriptor.err)); + private boolean contains(String source, String target, boolean isCaseSensitive) { String superString = source; String subString = target; @@ -95,5 +103,13 @@ protected void teardownMockedAppender() { Logger.getRootLogger().setLevel(originalLevel); Logger.getRootLogger().addAppender(originalAppender); } + + public static void runWithoutOutput(Runnable function) { + System.setOut(NULL); + System.setErr(NULL); + function.run(); + System.setOut(OUT); + System.setErr(ERR); + } } diff --git a/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java b/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java index af66652..96755d1 100644 --- a/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java +++ b/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java @@ -16,7 +16,7 @@ package com.yahoo.validatar.execution; -import com.yahoo.validatar.LogCaptor; +import com.yahoo.validatar.OutputCaptor; import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.Result; import com.yahoo.validatar.common.TypeSystem; @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; -public class EngineManagerTest extends LogCaptor { +public class EngineManagerTest extends OutputCaptor { private class MockFailingEngine implements Engine { public static final String ENGINE_NAME = "FAILER"; diff --git a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java new file mode 100644 index 0000000..e4e6701 --- /dev/null +++ b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java @@ -0,0 +1,46 @@ +package com.yahoo.validatar.execution.pig; + +import com.yahoo.validatar.common.Query; +import com.yahoo.validatar.common.TestSuite; +import com.yahoo.validatar.common.TypedObject; +import com.yahoo.validatar.parse.yaml.YAML; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.List; +import java.util.Map; + +import static com.yahoo.validatar.OutputCaptor.runWithoutOutput; + +public class StyTest { + @Test + public void testDefaults() { + Sty sty = new Sty(); + Assert.assertTrue(sty.setup(new String[0])); + Assert.assertEquals(sty.getName(), Sty.ENGINE_NAME); + } + + @Test + public void testFullQuery() throws FileNotFoundException { + File testFile = new File(getClass().getClassLoader().getResource("pig-tests/sample.yaml").getFile()); + TestSuite testSuite = new YAML().parse(new FileInputStream(testFile)); + Query query = testSuite.queries.stream().filter(q -> "Query".equals(q.name)).findAny().get(); + + Sty sty = new Sty(); + sty.setup(new String[0]); + runWithoutOutput(() -> sty.execute(query)); + + Assert.assertFalse(query.failed()); + Map> result = query.getResult().getColumns(); + Assert.assertEquals(result.size(), 2); + List names = result.get("Query.name"); + List totals = result.get("Query.total"); + Assert.assertEquals(names.size(), 1); + Assert.assertEquals(totals.size(), 1); + Assert.assertEquals(names.get(0).data, String.valueOf("foo")); + Assert.assertEquals(totals.get(0).data, Long.valueOf(3)); + } +} diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties index b82effb..941ca0f 100644 --- a/src/test/resources/log4j.properties +++ b/src/test/resources/log4j.properties @@ -1,6 +1,6 @@ # Root logger option log4j.rootLogger= FATAL, stdout - + # Direct log messages to stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out diff --git a/src/test/resources/pig-tests/data.txt b/src/test/resources/pig-tests/data.txt new file mode 100644 index 0000000..6100d00 --- /dev/null +++ b/src/test/resources/pig-tests/data.txt @@ -0,0 +1,9 @@ +foo,12 +bar,1 +foo,1 +foo,-4 +bar,0 +baz,-26 +foo,-8 +baz,19 +foo,2 diff --git a/src/test/resources/pig-tests/sample.yaml b/src/test/resources/pig-tests/sample.yaml new file mode 100644 index 0000000..77b8f52 --- /dev/null +++ b/src/test/resources/pig-tests/sample.yaml @@ -0,0 +1,23 @@ +--- +name: Pig test +description: Testing Validatar +queries: + - name: Query + engine: pig + value: "a = LOAD 'src/test/resources/pig-tests/data.txt' USING PigStorage(',') AS (name:chararray, count:long); + b = GROUP a BY name; + c = FOREACH b GENERATE + group AS name, + SUM(a.count) AS total; + d = ORDER c BY total DESC; + e = LIMIT d 1;" + metadata: + - key: output-alias + value: e + - key: exec-type + value: local +tests: + - name: Simple Test + description: Why I am testing this. + asserts: +... \ No newline at end of file From 7d084e8f0920b6541aefe6dcd184932f16c50207 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Wed, 16 Sep 2015 08:45:06 -0700 Subject: [PATCH 07/27] Adding trial clover --- pom.xml | 11 ++++++++++- .../java/com/yahoo/validatar/execution/pig/Sty.java | 8 ++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 31d893c..efd30dd 100644 --- a/pom.xml +++ b/pom.xml @@ -153,6 +153,15 @@ + com.atlassian.maven.plugins + maven-clover2-plugin + 4.0.5 + + false + + + + org.eluder.coveralls diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 5c854d6..265edfe 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -165,7 +165,7 @@ private void populateRow(Tuple row, List metadata, Result result) t } } - private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { + TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { byte type = detail.type; switch (type) { case DataType.BOOLEAN: @@ -198,19 +198,19 @@ private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecE } } - private List getFieldDetails(Schema schema) { + List getFieldDetails(Schema schema) { if (schema == null) { return Collections.emptyList(); } return schema.getFields().stream().map(f -> new FieldDetail(f.alias, f.type)).collect(Collectors.toList()); } - private Optional getKey(Map metadata, String key) { + Optional getKey(Map metadata, String key) { String value = metadata.get(key); return value == null || value.isEmpty() ? Optional.empty() : Optional.of(value); } - private Properties getProperties(OptionSet options) { + Properties getProperties(OptionSet options) { List settings = (List) options.valuesOf(PIG_SETTING); Properties properties = new Properties(); for (String setting : settings) { From c2a84700a878d930c192df17e8580382d33b98f5 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Wed, 16 Sep 2015 08:59:16 -0700 Subject: [PATCH 08/27] Testing if latest coveralls maven supports clover --- .travis.yml | 2 +- Makefile | 6 +++--- pom.xml | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94abd93..22e7aa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ jdk: - oraclejdk8 after_success: -- mvn clean cobertura:cobertura coveralls:report +- mvn clean clover2:setup test clover2:aggregate clover2:clover coveralls:report - test "${TRAVIS_PULL_REQUEST}" == "false" && test "${TRAVIS_TAG}" != "" && mvn deploy --settings travis/settings.xml cache: diff --git a/Makefile b/Makefile index 6de7ce7..203de03 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: full full: - mvn clean checkstyle:check cobertura:cobertura javadoc:jar package + mvn clean checkstyle:check javadoc:jar package clean: mvn clean @@ -16,13 +16,13 @@ release: mvn -B release:prepare release:clean coverage: - mvn clean cobertura:cobertura + mvn clean clover2:setup test clover2:aggregate clover2:clover doc: mvn clean javadoc:javadoc see-coverage: coverage - cd target/site/cobertura; python -m SimpleHTTPServer + cd target/site/clover; python -m SimpleHTTPServer see-doc: doc cd target/site/apidocs; python -m SimpleHTTPServer diff --git a/pom.xml b/pom.xml index efd30dd..3789f8f 100644 --- a/pom.xml +++ b/pom.xml @@ -157,6 +157,9 @@ maven-clover2-plugin 4.0.5 + + com/yahoo/validatar/assertion/Grammar*.java + false @@ -182,7 +185,7 @@ org.eluder.coveralls coveralls-maven-plugin - 3.1.0 + 4.0.0 From 4331ce5458fdfc3b240174522c5b28ba026a3020 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Wed, 16 Sep 2015 09:43:41 -0700 Subject: [PATCH 09/27] Adding default test --- .../yahoo/validatar/execution/pig/Sty.java | 23 +++++-- .../validatar/execution/pig/StyTest.java | 66 +++++++++++++++++-- src/test/resources/pig-tests/sample.yaml | 5 ++ 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 265edfe..52f4353 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -124,7 +124,7 @@ public void execute(Query query) { String alias = getKey(queryMetadata, METADATA_ALIAS_KEY).orElse(defaultOutputAlias); log.info("Running " + queryName + " for alias " + alias + ": " + queryValue); try { - PigServer server = new PigServer(execType, properties); + PigServer server = getPigServer(execType); server.registerScript(new ByteArrayInputStream(queryValue.getBytes())); Iterator queryResults = server.openIterator(alias); Result result = query.createResults(); @@ -165,7 +165,7 @@ private void populateRow(Tuple row, List metadata, Result result) t } } - TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { + private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { byte type = detail.type; switch (type) { case DataType.BOOLEAN: @@ -198,19 +198,22 @@ TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException } } - List getFieldDetails(Schema schema) { + private List getFieldDetails(Schema schema) { if (schema == null) { return Collections.emptyList(); } return schema.getFields().stream().map(f -> new FieldDetail(f.alias, f.type)).collect(Collectors.toList()); } - Optional getKey(Map metadata, String key) { + private Optional getKey(Map metadata, String key) { + if (metadata == null) { + return Optional.empty(); + } String value = metadata.get(key); return value == null || value.isEmpty() ? Optional.empty() : Optional.of(value); } - Properties getProperties(OptionSet options) { + private Properties getProperties(OptionSet options) { List settings = (List) options.valuesOf(PIG_SETTING); Properties properties = new Properties(); for (String setting : settings) { @@ -224,4 +227,14 @@ Properties getProperties(OptionSet options) { return properties; } + /** + * Only used for testing. + * + * @param execType The execType for Pig + * @return A {@link PigServer} object + * @throws IOException + */ + PigServer getPigServer(String execType) throws IOException { + return new PigServer(execType, properties); + } } diff --git a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java index e4e6701..d7dc161 100644 --- a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java +++ b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java @@ -4,33 +4,85 @@ import com.yahoo.validatar.common.TestSuite; import com.yahoo.validatar.common.TypedObject; import com.yahoo.validatar.parse.yaml.YAML; +import org.apache.pig.PigServer; +import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.mockito.Mockito; import org.testng.Assert; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Properties; import static com.yahoo.validatar.OutputCaptor.runWithoutOutput; public class StyTest { + private String[] defaults = {"--pig-exec-type", "local"}; + + private Sty sty; + + private PigServer getServer(String execType, Properties properties) throws IOException { + PigServer server = new PigServer(execType, properties); + PigServer spiedServer = Mockito.spy(server); + return spiedServer; + } + + private PigServer withMockSchema(PigServer server, Schema schema) throws IOException { + Mockito.doReturn(schema).when(server).dumpSchema(Mockito.anyString()); + return server; + } + + private Sty getSty(PigServer server) throws IOException { + Sty sty = new Sty(); + Sty spiedSty = Mockito.spy(sty); + Mockito.doReturn(server).when(spiedSty).getPigServer(Mockito.anyString()); + return spiedSty; + } + + private Query getQueryFrom(String file, String name) throws FileNotFoundException { + File testFile = new File(getClass().getClassLoader().getResource(file).getFile()); + TestSuite testSuite = new YAML().parse(new FileInputStream(testFile)); + return testSuite.queries.stream().filter(q -> name.equals(q.name)).findAny().get(); + } + + @BeforeMethod + public void setup() throws IOException { + sty = getSty(getServer("local", new Properties())); + } + @Test public void testDefaults() { - Sty sty = new Sty(); Assert.assertTrue(sty.setup(new String[0])); Assert.assertEquals(sty.getName(), Sty.ENGINE_NAME); } @Test - public void testFullQuery() throws FileNotFoundException { - File testFile = new File(getClass().getClassLoader().getResource("pig-tests/sample.yaml").getFile()); - TestSuite testSuite = new YAML().parse(new FileInputStream(testFile)); - Query query = testSuite.queries.stream().filter(q -> "Query".equals(q.name)).findAny().get(); + public void testDefaultedQuery() throws FileNotFoundException { + Query query = getQueryFrom("pig-tests/sample.yaml", "Defaults"); + + sty.setup(defaults); + runWithoutOutput(() -> sty.execute(query)); + + Assert.assertFalse(query.failed()); + Map> result = query.getResult().getColumns(); + Assert.assertEquals(result.size(), 2); + List names = result.get("Defaults.name"); + List counts = result.get("Defaults.count"); + Assert.assertEquals(names.size(), 1); + Assert.assertEquals(counts.size(), 1); + Assert.assertEquals(names.get(0).data, String.valueOf("baz")); + Assert.assertEquals(counts.get(0).data, Long.valueOf(-26)); + } + + @Test + public void testMetadataQuery() throws FileNotFoundException { + Query query = getQueryFrom("pig-tests/sample.yaml", "Query"); - Sty sty = new Sty(); - sty.setup(new String[0]); runWithoutOutput(() -> sty.execute(query)); Assert.assertFalse(query.failed()); diff --git a/src/test/resources/pig-tests/sample.yaml b/src/test/resources/pig-tests/sample.yaml index 77b8f52..4160212 100644 --- a/src/test/resources/pig-tests/sample.yaml +++ b/src/test/resources/pig-tests/sample.yaml @@ -16,6 +16,11 @@ queries: value: e - key: exec-type value: local + - name: Defaults + engine: pig + value: "a = LOAD 'src/test/resources/pig-tests/data.txt' USING PigStorage(',') AS (name:chararray, count:long); + b = ORDER a BY count; + validatar_results = LIMIT b 1;" tests: - name: Simple Test description: Why I am testing this. From 4448b1b037d6bc12efddeb1c7c1874badc71959e Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 17 Sep 2015 00:50:26 -0700 Subject: [PATCH 10/27] Fixing Collectors toMap issue with null values. Adding all Pig tests except type specific checks --- .../com/yahoo/validatar/common/Query.java | 8 +- .../yahoo/validatar/execution/pig/Sty.java | 19 +-- .../validatar/execution/pig/StyTest.java | 137 +++++++++++++++++- 3 files changed, 146 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/common/Query.java b/src/main/java/com/yahoo/validatar/common/Query.java index be6ee4b..9da3514 100644 --- a/src/main/java/com/yahoo/validatar/common/Query.java +++ b/src/main/java/com/yahoo/validatar/common/Query.java @@ -16,9 +16,9 @@ package com.yahoo.validatar.common; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; public class Query extends Executable { public String name; @@ -69,6 +69,10 @@ public Map getMetadata() { if (metadata == null) { return null; } - return metadata.stream().collect(Collectors.toMap(entry -> entry.key, entry -> entry.value)); + Map map = new HashMap<>(); + for (Metadata entry : metadata) { + map.put(entry.key, entry.value); + } + return map; } } diff --git a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java index 52f4353..dbafab2 100644 --- a/src/main/java/com/yahoo/validatar/execution/pig/Sty.java +++ b/src/main/java/com/yahoo/validatar/execution/pig/Sty.java @@ -166,6 +166,9 @@ private void populateRow(Tuple row, List metadata, Result result) t } private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException { + if (data == null) { + return null; + } byte type = detail.type; switch (type) { case DataType.BOOLEAN: @@ -185,15 +188,8 @@ private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecE case DataType.BIGINTEGER: case DataType.BIGDECIMAL: return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type)); - case DataType.TUPLE: - case DataType.BAG: - case DataType.MAP: - case DataType.INTERNALMAP: - case DataType.GENERIC_WRITABLECOMPARABLE: - case DataType.ERROR: - case DataType.UNKNOWN: - case DataType.NULL: default: + //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else return null; } } @@ -227,13 +223,6 @@ private Properties getProperties(OptionSet options) { return properties; } - /** - * Only used for testing. - * - * @param execType The execType for Pig - * @return A {@link PigServer} object - * @throws IOException - */ PigServer getPigServer(String execType) throws IOException { return new PigServer(execType, properties); } diff --git a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java index d7dc161..4815abf 100644 --- a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java +++ b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java @@ -1,10 +1,14 @@ package com.yahoo.validatar.execution.pig; +import com.yahoo.validatar.common.Metadata; import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.TestSuite; import com.yahoo.validatar.common.TypedObject; import com.yahoo.validatar.parse.yaml.YAML; import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.data.DataType; +import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.mockito.Mockito; import org.testng.Assert; @@ -15,6 +19,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; @@ -26,17 +31,49 @@ public class StyTest { private Sty sty; + private Schema.FieldSchema makeFieldSchema(String alias, byte type) { + return new Schema.FieldSchema(alias, type); + } + + private Tuple makeTuple(Object... contents) throws ExecException { + Tuple mocked = Mockito.mock(Tuple.class); + if (contents == null) { + Mockito.when(mocked.get(0)).thenReturn(null); + return mocked; + } + for (int i = 0; i < contents.length; ++i) { + Mockito.when(mocked.get(i)).thenReturn(contents[i]); + } + return mocked; + } + + private PigServer getServer() throws IOException { + return getServer("local", new Properties()); + } + private PigServer getServer(String execType, Properties properties) throws IOException { PigServer server = new PigServer(execType, properties); PigServer spiedServer = Mockito.spy(server); return spiedServer; } + private Schema getSchema(Schema.FieldSchema... schema) { + Schema mocked = Mockito.mock(Schema.class); + Mockito.when(mocked.getFields()).thenReturn(Arrays.asList(schema)); + return mocked; + } + private PigServer withMockSchema(PigServer server, Schema schema) throws IOException { Mockito.doReturn(schema).when(server).dumpSchema(Mockito.anyString()); return server; } + private PigServer withMockResult(PigServer server, Tuple... results) throws IOException { + Mockito.doReturn(results == null ? null : Arrays.asList(results).iterator()) + .when(server).openIterator(Mockito.anyString()); + return server; + } + private Sty getSty(PigServer server) throws IOException { Sty sty = new Sty(); Sty spiedSty = Mockito.spy(sty); @@ -44,6 +81,10 @@ private Sty getSty(PigServer server) throws IOException { return spiedSty; } + private Sty getSty() { + return new Sty(); + } + private Query getQueryFrom(String file, String name) throws FileNotFoundException { File testFile = new File(getClass().getClassLoader().getResource(file).getFile()); TestSuite testSuite = new YAML().parse(new FileInputStream(testFile)); @@ -52,7 +93,8 @@ private Query getQueryFrom(String file, String name) throws FileNotFoundExceptio @BeforeMethod public void setup() throws IOException { - sty = getSty(getServer("local", new Properties())); + sty = getSty(getServer()); + sty.setup(defaults); } @Test @@ -61,10 +103,75 @@ public void testDefaults() { Assert.assertEquals(sty.getName(), Sty.ENGINE_NAME); } + @Test + public void testNullQuery() { + Query query = new Query(); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertTrue(query.failed()); + Assert.assertTrue(query.getMessages().get(0).contains("java.lang.NullPointerException")); + } + + @Test + public void testNullSchema() throws IOException { + Query query = new Query(); + query.value = ""; + sty = getSty(withMockResult(withMockSchema(getServer(), null), (Tuple[]) null)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertTrue(query.failed()); + Assert.assertTrue(query.getMessages().get(0).contains("No metadata of columns found")); + } + + @Test + public void testNullTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fake = getSchema(makeFieldSchema("a", DataType.CHARARRAY)); + sty = getSty(withMockResult(withMockSchema(getServer(), fake), null, null)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List result = query.getResult().getColumn("a"); + Assert.assertNotNull(result); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testNullValueInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.CHARARRAY)); + Tuple fakeTuple = makeTuple((Object[]) null); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List result = query.getResult().getColumn("a"); + Assert.assertNotNull(result); + Assert.assertEquals(result.size(), 1); + Assert.assertNull(result.get(0)); + } + + @Test + public void testNullTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.NULL)); + Tuple fakeTuple = makeTuple("something"); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List result = query.getResult().getColumn("a"); + Assert.assertNotNull(result); + Assert.assertEquals(result.size(), 1); + Assert.assertNull(result.get(0)); + } + @Test public void testDefaultedQuery() throws FileNotFoundException { Query query = getQueryFrom("pig-tests/sample.yaml", "Defaults"); + // Loading + Sty sty = getSty(); sty.setup(defaults); runWithoutOutput(() -> sty.execute(query)); @@ -79,6 +186,34 @@ public void testDefaultedQuery() throws FileNotFoundException { Assert.assertEquals(counts.get(0).data, Long.valueOf(-26)); } + @Test + public void testBadProperties() throws IOException { + String[] problemSetting = {"--pig-setting", "pig.tmpFileCompression:false", + "--pig-setting", "pig.maxCombinedSplitSize=1024"}; + Assert.assertTrue(sty.setup(problemSetting)); + + Query query = getQueryFrom("pig-tests/sample.yaml", "Defaults"); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + } + + @Test + public void testBadMetadataQuery() throws FileNotFoundException { + Query query = getQueryFrom("pig-tests/sample.yaml", "Defaults"); + // Null value + Metadata badOne = new Metadata(); + badOne.key = Sty.METADATA_ALIAS_KEY; + badOne.value = null; + // Empty value + Metadata badTwo = new Metadata(); + badOne.key = Sty.METADATA_ALIAS_KEY; + badOne.value = ""; + query.metadata = Arrays.asList(badOne, badTwo); + + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + } + @Test public void testMetadataQuery() throws FileNotFoundException { Query query = getQueryFrom("pig-tests/sample.yaml", "Query"); From 1a507403b6595bf42b87773edc1cb6d5cfcda1b3 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 17 Sep 2015 08:53:29 -0700 Subject: [PATCH 11/27] Adding type tests. Full coverage --- .../com/yahoo/validatar/common/Query.java | 5 +- .../validatar/execution/pig/StyTest.java | 129 ++++++++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/common/Query.java b/src/main/java/com/yahoo/validatar/common/Query.java index 9da3514..a9527d8 100644 --- a/src/main/java/com/yahoo/validatar/common/Query.java +++ b/src/main/java/com/yahoo/validatar/common/Query.java @@ -70,9 +70,8 @@ public Map getMetadata() { return null; } Map map = new HashMap<>(); - for (Metadata entry : metadata) { - map.put(entry.key, entry.value); - } + // default Collectors.toMap doesn't handle null values + metadata.stream().forEach(m -> map.put(m.key, m.value)); return map; } } diff --git a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java index 4815abf..5d962cc 100644 --- a/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java +++ b/src/test/java/com/yahoo/validatar/execution/pig/StyTest.java @@ -7,9 +7,11 @@ import com.yahoo.validatar.parse.yaml.YAML; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.data.DataByteArray; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.joda.time.DateTime; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.BeforeMethod; @@ -19,6 +21,9 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -27,6 +32,8 @@ import static com.yahoo.validatar.OutputCaptor.runWithoutOutput; public class StyTest { + public static final double EPSILON = 0.00001; + private String[] defaults = {"--pig-exec-type", "local"}; private Sty sty; @@ -101,6 +108,7 @@ public void setup() throws IOException { public void testDefaults() { Assert.assertTrue(sty.setup(new String[0])); Assert.assertEquals(sty.getName(), Sty.ENGINE_NAME); + runWithoutOutput(() -> sty.printHelp()); } @Test @@ -166,6 +174,127 @@ public void testNullTypeInTuple() throws IOException { Assert.assertNull(result.get(0)); } + @Test + public void testBooleanTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BOOLEAN)); + Tuple fakeTuple = makeTuple(Boolean.valueOf(false)); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List result = query.getResult().getColumn("a"); + Assert.assertNotNull(result); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.get(0).data, Boolean.valueOf(false)); + } + + @Test + public void testIntegerTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.INTEGER), + makeFieldSchema("b", DataType.LONG)); + Tuple fakeTuple = makeTuple(Integer.valueOf(42), Long.valueOf(84)); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List columnOne = query.getResult().getColumn("a"); + List columnTwo = query.getResult().getColumn("b"); + Assert.assertNotNull(columnOne); + Assert.assertEquals(columnOne.size(), 1); + Assert.assertEquals(columnOne.get(0).data, Long.valueOf(42)); + Assert.assertNotNull(columnTwo); + Assert.assertEquals(columnTwo.size(), 1); + Assert.assertEquals(columnTwo.get(0).data, Long.valueOf(84)); + } + + @Test + public void testFloatTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.FLOAT), + makeFieldSchema("b", DataType.DOUBLE)); + Tuple fakeTuple = makeTuple(Float.valueOf(2.1f), Double.valueOf(4.2)); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List columnOne = query.getResult().getColumn("a"); + List columnTwo = query.getResult().getColumn("b"); + Assert.assertNotNull(columnOne); + Assert.assertEquals(columnOne.size(), 1); + Assert.assertTrue(Math.abs((Double) columnOne.get(0).data - Double.valueOf(2.1)) < EPSILON); + Assert.assertNotNull(columnTwo); + Assert.assertEquals(columnTwo.size(), 1); + Assert.assertTrue(Math.abs((Double) columnTwo.get(0).data - Double.valueOf(4.2)) < EPSILON); + } + + @Test + public void testBigNumericTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BIGINTEGER), + makeFieldSchema("b", DataType.BIGDECIMAL)); + Tuple fakeTuple = makeTuple(new BigInteger("42"), new BigDecimal("42.1")); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List columnOne = query.getResult().getColumn("a"); + List columnTwo = query.getResult().getColumn("b"); + Assert.assertNotNull(columnOne); + Assert.assertEquals(columnOne.size(), 1); + Assert.assertEquals(columnOne.get(0).data, new BigDecimal("42")); + Assert.assertNotNull(columnTwo); + Assert.assertEquals(columnTwo.size(), 1); + Assert.assertEquals(columnTwo.get(0).data, new BigDecimal("42.1")); + } + + @Test + public void testStringTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BYTE), + makeFieldSchema("b", DataType.BYTEARRAY), + makeFieldSchema("c", DataType.CHARARRAY)); + Tuple fakeTuple = makeTuple(Byte.valueOf("1"), new DataByteArray("foo".getBytes()), "bar"); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List columnOne = query.getResult().getColumn("a"); + List columnTwo = query.getResult().getColumn("b"); + List columnThree = query.getResult().getColumn("c"); + Assert.assertNotNull(columnOne); + Assert.assertEquals(columnOne.size(), 1); + Assert.assertEquals(columnOne.get(0).data, "1"); + Assert.assertNotNull(columnTwo); + Assert.assertEquals(columnTwo.size(), 1); + Assert.assertEquals(columnTwo.get(0).data, "foo"); + Assert.assertNotNull(columnThree); + Assert.assertEquals(columnThree.size(), 1); + Assert.assertEquals(columnThree.get(0).data, "bar"); + } + + @Test + public void testDateTypeInTuple() throws IOException { + Query query = new Query(); + query.value = ""; + Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.DATETIME)); + Tuple fakeTuple = makeTuple(new DateTime(142151414341L)); + + sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple)); + runWithoutOutput(() -> sty.execute(query)); + Assert.assertFalse(query.failed()); + List result = query.getResult().getColumn("a"); + Assert.assertNotNull(result); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.get(0).data, new Timestamp(142151414341L)); + } + @Test public void testDefaultedQuery() throws FileNotFoundException { Query query = getQueryFrom("pig-tests/sample.yaml", "Defaults"); From d79b89222a5439639d3ec0fd428918493c14189c Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 17 Sep 2015 22:44:42 -0700 Subject: [PATCH 12/27] Adding ParseManager tests --- .../yahoo/validatar/parse/ParseManager.java | 4 +- .../validatar/parse/ParseManagerTest.java | 94 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java diff --git a/src/main/java/com/yahoo/validatar/parse/ParseManager.java b/src/main/java/com/yahoo/validatar/parse/ParseManager.java index 45ac9b3..e3ffd54 100644 --- a/src/main/java/com/yahoo/validatar/parse/ParseManager.java +++ b/src/main/java/com/yahoo/validatar/parse/ParseManager.java @@ -91,7 +91,6 @@ private Stream getFiles(File path) { */ public static void expandParameters(List suites, Map parameterMap) { Objects.requireNonNull(suites); - Objects.requireNonNull(parameterMap); suites.stream().filter(Objects::nonNull).map(s -> s.queries) .flatMap(Collection::stream).filter(Objects::nonNull) @@ -107,6 +106,7 @@ public static void expandParameters(List suites, Map */ public static void expandParameters(Query query, Map parameterMap) { Objects.requireNonNull(query); + Objects.requireNonNull(parameterMap); Matcher matcher = REGEX.matcher(query.value); StringBuffer newQuery = new StringBuffer(); while (matcher.find()) { @@ -138,7 +138,7 @@ protected TestSuite getTestSuite(File path) { } try { return parser.parse(new FileInputStream(path)); - } catch (FileNotFoundException e) { + } catch (Exception e) { log.error(e); return null; } diff --git a/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java b/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java new file mode 100644 index 0000000..5e016b9 --- /dev/null +++ b/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java @@ -0,0 +1,94 @@ +package com.yahoo.validatar.parse; + +import com.yahoo.validatar.common.Query; +import com.yahoo.validatar.common.TestSuite; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +public class ParseManagerTest { + @Test + public void testFailLoadOfNonFile() { + ParseManager manager = new ParseManager(); + TestSuite testSuite = manager.getTestSuite(new File("src/test/resources")); + Assert.assertNull(testSuite); + } + + @Test + public void testFailLoadOfUnknownFile() { + ParseManager manager = new ParseManager(); + TestSuite testSuite = manager.getTestSuite(new File("src/test/resources/log4j.properties")); + Assert.assertNull(testSuite); + } + + @Test + public void testFailLoadOfBadExtensionFile() { + ParseManager manager = new ParseManager(); + TestSuite testSuite = manager.getTestSuite(new File("src/test/resources/log4j.properties")); + Assert.assertNull(testSuite); + } + + @Test + public void testFailLoadOfNoExtensionFile() { + ParseManager manager = new ParseManager(); + TestSuite testSuite = manager.getTestSuite(new File("LICENSE")); + Assert.assertNull(testSuite); + } + + @Test + public void testFailLoadOfDisappearingFile() { + ParseManager manager = new ParseManager(); + File mocked = Mockito.mock(File.class); + Mockito.when(mocked.isFile()).thenReturn(true); + Mockito.when(mocked.getName()).thenReturn("foo.yaml"); + Mockito.when(mocked.getPath()).thenReturn(null); + TestSuite testSuite = manager.getTestSuite(mocked); + Assert.assertNull(testSuite); + } + + @Test + public void testFailLoadOfNullPath() { + ParseManager manager = new ParseManager(); + List loaded = manager.load(null); + Assert.assertTrue(loaded.isEmpty()); + } + + @Test + public void testFailLoadOfEmptyDirectory() { + ParseManager manager = new ParseManager(); + File mocked = Mockito.mock(File.class); + Mockito.when(mocked.isFile()).thenReturn(false); + Mockito.when(mocked.listFiles()).thenReturn(null); + List loaded = manager.load(mocked); + Assert.assertTrue(loaded.isEmpty()); + } + + @Test + public void testLoadOfDirectory() { + ParseManager manager = new ParseManager(); + List loaded = manager.load(new File("src/test/resources/sample-tests")); + Assert.assertEquals(loaded.size(), 3); + } + + @Test + public void testFailExpansion() { + Query query = new Query(); + query.value = "${var}"; + ParseManager manager = new ParseManager(); + manager.expandParameters(query, Collections.emptyMap()); + Assert.assertEquals(query.value, "${var}"); + } + + @Test + public void testExpansion() { + Query query = new Query(); + query.value = "${var}"; + ParseManager manager = new ParseManager(); + manager.expandParameters(query, Collections.singletonMap("var", "foo")); + Assert.assertEquals(query.value, "foo"); + } +} From 166e23d1e745e34ca225380b6bd349a57340d14f Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 17 Sep 2015 23:23:22 -0700 Subject: [PATCH 13/27] Adding Format tests --- .../validatar/execution/EngineManager.java | 2 +- .../yahoo/validatar/report/FormatManager.java | 12 +++---- .../validatar/report/FormatManagerTest.java | 32 ++++++++++++++++++- .../report/junit/JUnitFormatterTest.java | 12 +++---- src/test/resources/ExpectedJUnitOutput.xml | 5 ++- .../resources/sample-tests/simple-tests.yaml | 4 +++ 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/execution/EngineManager.java b/src/main/java/com/yahoo/validatar/execution/EngineManager.java index 3cbc3f2..f7db265 100644 --- a/src/main/java/com/yahoo/validatar/execution/EngineManager.java +++ b/src/main/java/com/yahoo/validatar/execution/EngineManager.java @@ -110,7 +110,7 @@ public EngineManager(String[] arguments) { * * @param engines A list of engines to use as the engines to work with. */ - protected void setEngines(List engines) { + void setEngines(List engines) { List all = engines == null ? Collections.emptyList() : engines; this.engines = all.stream().collect(Collectors.toMap(Engine::getName, WorkingEngine::new)); } diff --git a/src/main/java/com/yahoo/validatar/report/FormatManager.java b/src/main/java/com/yahoo/validatar/report/FormatManager.java index e6f6fbd..575bdb8 100644 --- a/src/main/java/com/yahoo/validatar/report/FormatManager.java +++ b/src/main/java/com/yahoo/validatar/report/FormatManager.java @@ -59,12 +59,15 @@ public class FormatManager implements Helpable { * @param arguments An array of parameters of the form [--param1 value1 --param2 value2...] */ public FormatManager(String[] arguments) { + loadFormatters(); String name = (String) parser.parse(arguments).valueOf(REPORT_FORMAT); - formatterToUse = findFormatter(name, arguments); + formatterToUse = availableFormatters.get(name); + if (formatterToUse == null) { printHelp(); throw new RuntimeException("Could not find a formatter to use"); } + if (!formatterToUse.setup(arguments)) { formatterToUse.printHelp(); throw new RuntimeException("Could not initialize the requested formatter"); @@ -80,17 +83,13 @@ protected void setFormatterToUse(Formatter formatter) { formatterToUse = formatter; } - private Formatter findFormatter(String name, String[] arguments) { + private void loadFormatters() { Reflections reflections = new Reflections("com.yahoo.validatar.report"); Set> subTypes = reflections.getSubTypesOf(Formatter.class); availableFormatters = new HashMap<>(); - Formatter searchingFor = null; for (Class formatterClass : subTypes) { try { Formatter formatter = formatterClass.newInstance(); - if (name.equals(formatter.getName())) { - searchingFor = formatter; - } availableFormatters.put(formatter.getName(), formatter); log.info("Setup formatter " + formatter.getName()); } catch (InstantiationException e) { @@ -99,7 +98,6 @@ private Formatter findFormatter(String name, String[] arguments) { log.info("Illegal access of " + formatterClass + " " + e); } } - return searchingFor; } /** diff --git a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java index 814629f..2839ed5 100644 --- a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java +++ b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java @@ -48,7 +48,31 @@ public void writeReport(List testSuites) throws IOException { @Override public String getName() { - return "MockFormatter"; + return "MockFormat"; + } + } + + // Used for tests + public static class FailingFormatter implements Formatter { + public FailingFormatter() { + } + + @Override + public boolean setup(String[] arguments) { + return false; + } + + @Override + public void printHelp() { + } + + @Override + public void writeReport(List testSuites) throws IOException { + } + + @Override + public String getName() { + return "FailingFormat"; } } @@ -66,6 +90,12 @@ public void testConstructorAndFindFormatterExceptionNoFormatterFound() throws Fi System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.err))); } + @Test(expectedExceptions = {RuntimeException.class}) + public void testFailFormatterSetup() throws IOException { + String[] args = {"--report-format", "FailingFormat"}; + FormatManager manager = new FormatManager(args); + } + @Test public void testWriteReport() throws IOException { String[] args = {}; diff --git a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java index 917884d..204a9d5 100644 --- a/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java +++ b/src/test/java/com/yahoo/validatar/report/junit/JUnitFormatterTest.java @@ -16,6 +16,7 @@ package com.yahoo.validatar.report.junit; +import com.yahoo.validatar.common.Query; import com.yahoo.validatar.common.TestSuite; import com.yahoo.validatar.parse.ParseManager; import org.dom4j.Document; @@ -45,13 +46,10 @@ public void testWriteReport() throws IOException, org.dom4j.DocumentException { Assert.assertEquals(testSuites.size(), 3); - // Ensure that we are modifying the 'Simple examples' test suite - com.yahoo.validatar.common.Test test; - if (testSuites.get(0).name.equals("Simple examples")) { - test = testSuites.get(0).tests.get(1); - } else { - test = testSuites.get(1).tests.get(1); - } + TestSuite simpleExamples = testSuites.stream().filter(s -> "Simple examples".equals(s.name)).findFirst().get(); + Query failingQuery = simpleExamples.queries.get(2); + failingQuery.setFailure("Query had a typo"); + com.yahoo.validatar.common.Test test = simpleExamples.tests.get(1); test.setFailed(); test.addMessage("Sample fail message"); diff --git a/src/test/resources/ExpectedJUnitOutput.xml b/src/test/resources/ExpectedJUnitOutput.xml index 2f8a90d..09080e2 100644 --- a/src/test/resources/ExpectedJUnitOutput.xml +++ b/src/test/resources/ExpectedJUnitOutput.xml @@ -5,9 +5,12 @@ - + + + Query had a typo + Description: Long description of the second query; Failed asserts: Sample fail message diff --git a/src/test/resources/sample-tests/simple-tests.yaml b/src/test/resources/sample-tests/simple-tests.yaml index 291134c..1098961 100644 --- a/src/test/resources/sample-tests/simple-tests.yaml +++ b/src/test/resources/sample-tests/simple-tests.yaml @@ -12,6 +12,10 @@ queries: value: "SELECT SUM(CASE WHEN y = true THEN 1 ELSE 0 END) as y_count FROM sample_data TABLESAMPLE(0.1 percent) WHERE dt=${DATE}" + - name: FAIL + engine: pig + value: "LOAD '${FILE}' USING PigStorage(','); + BAD_TYPO_HERE" tests: - name: Checking alpha description: Looks at count output. From d17d9edf53345b8aa417dfce17c40bc4ddf51054 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 17 Sep 2015 23:41:00 -0700 Subject: [PATCH 14/27] Adding reflection tests and exception test for printing --- .../yahoo/validatar/common/HelpableTest.java | 18 +++++++++++ .../validatar/parse/ParseManagerTest.java | 30 ++++++++++++++++++ .../validatar/report/FormatManagerTest.java | 31 +++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 src/test/java/com/yahoo/validatar/common/HelpableTest.java diff --git a/src/test/java/com/yahoo/validatar/common/HelpableTest.java b/src/test/java/com/yahoo/validatar/common/HelpableTest.java new file mode 100644 index 0000000..deb8687 --- /dev/null +++ b/src/test/java/com/yahoo/validatar/common/HelpableTest.java @@ -0,0 +1,18 @@ +package com.yahoo.validatar.common; + +import com.yahoo.validatar.OutputCaptor; +import joptsimple.OptionParser; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.OutputStream; + +public class HelpableTest { + @Test(expectedExceptions = {RuntimeException.class}) + public void testFailPrintHelp() throws IOException { + OptionParser mocked = Mockito.mock(OptionParser.class); + Mockito.doThrow(new IOException()).when(mocked).printHelpOn(Mockito.any(OutputStream.class)); + OutputCaptor.runWithoutOutput(() -> Helpable.printHelp("", mocked)); + } +} diff --git a/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java b/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java index 5e016b9..e1bd396 100644 --- a/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java +++ b/src/test/java/com/yahoo/validatar/parse/ParseManagerTest.java @@ -7,10 +7,40 @@ import org.testng.annotations.Test; import java.io.File; +import java.io.InputStream; import java.util.Collections; import java.util.List; public class ParseManagerTest { + // Reflection will load these classes and test the behavior of the code when handling them. + private class UninstantiableParser implements Parser { + @Override + public TestSuite parse(InputStream data) { + return null; + } + @Override + public String getName() { + return "Uninstantiable"; + } + } + + public static class IllegalAccessParser implements Parser { + public IllegalAccessParser() throws IllegalAccessException { + throw new IllegalAccessException(); + } + + @Override + public TestSuite parse(InputStream data) { + return null; + } + + @Override + public String getName() { + return "IllegalAccess"; + } + } + + @Test public void testFailLoadOfNonFile() { ParseManager manager = new ParseManager(); diff --git a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java index 2839ed5..8be5c3a 100644 --- a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java +++ b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java @@ -16,6 +16,7 @@ package com.yahoo.validatar.report; +import com.yahoo.validatar.OutputCaptor; import com.yahoo.validatar.common.TestSuite; import org.testng.Assert; import org.testng.annotations.Test; @@ -75,6 +76,30 @@ public String getName() { return "FailingFormat"; } } + // Used for tests + public static class IllegalAccessFormatter implements Formatter { + public IllegalAccessFormatter() throws IllegalAccessException{ + throw new IllegalAccessException(); + } + + @Override + public boolean setup(String[] arguments) { + return false; + } + + @Override + public void printHelp() { + } + + @Override + public void writeReport(List testSuites) throws IOException { + } + + @Override + public String getName() { + return "IllegalAccessFormat"; + } + } @Test public void testConstructorAndFindFormatterExceptionNoFormatterFound() throws FileNotFoundException { @@ -96,6 +121,12 @@ public void testFailFormatterSetup() throws IOException { FormatManager manager = new FormatManager(args); } + @Test(expectedExceptions = {RuntimeException.class}) + public void testFailInstantiation() throws IOException { + String[] args = {"--report-format", "IllegalAccessFormat"}; + OutputCaptor.runWithoutOutput(() -> new FormatManager(args)); + } + @Test public void testWriteReport() throws IOException { String[] args = {}; From d2a70c1acda816714ffb1bf5c1313aabe7b4963e Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 00:33:08 -0700 Subject: [PATCH 15/27] Removing unnecessary defaults in TypeSystem. Cannot test two lines in dispatch without sacrificing code quality. --- .../yahoo/validatar/common/TypeSystem.java | 11 ++++------ .../validatar/common/TypeSystemTest.java | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/yahoo/validatar/common/TypeSystem.java b/src/main/java/com/yahoo/validatar/common/TypeSystem.java index 3073180..bdb5f2f 100644 --- a/src/main/java/com/yahoo/validatar/common/TypeSystem.java +++ b/src/main/java/com/yahoo/validatar/common/TypeSystem.java @@ -20,6 +20,7 @@ import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.function.BinaryOperator; import java.util.function.UnaryOperator; @@ -60,7 +61,7 @@ public enum UnaryOperation { * Exceptions: * Timestamp to and from Long will do a millis since epoch */ - private interface Operations { + public interface Operations { default TypedObject add(TypedObject first, TypedObject second) { return null; } @@ -98,6 +99,7 @@ default TypedObject cast(TypedObject object) { } default BinaryOperator dispatch(BinaryOperation operator) { + Objects.requireNonNull(operator); switch (operator) { case ADD: return this::add; @@ -119,6 +121,7 @@ default BinaryOperator dispatch(BinaryOperation operator) { } default UnaryOperator dispatch(UnaryOperation operator) { + Objects.requireNonNull(operator); switch (operator) { case NEGATE: return this::negate; @@ -166,7 +169,6 @@ public TypedObject cast(TypedObject object) { case DOUBLE: case DECIMAL: case BOOLEAN: - default: return null; } object.type = Type.LONG; @@ -204,7 +206,6 @@ public TypedObject cast(TypedObject object) { case DECIMAL: case BOOLEAN: case TIMESTAMP: - default: return null; } object.type = Type.DOUBLE; @@ -250,7 +251,6 @@ public TypedObject cast(TypedObject object) { object.data = BigDecimal.valueOf(((Timestamp) object.data).getTime()); break; case BOOLEAN: - default: return null; } object.type = Type.DECIMAL; @@ -282,7 +282,6 @@ public TypedObject cast(TypedObject object) { case DOUBLE: case DECIMAL: case TIMESTAMP: - default: return null; } object.type = Type.BOOLEAN; @@ -312,7 +311,6 @@ public TypedObject cast(TypedObject object) { object.data = ((Boolean) object.data).toString(); break; case TIMESTAMP: - default: return null; } object.type = Type.STRING; @@ -352,7 +350,6 @@ public TypedObject cast(TypedObject object) { case DOUBLE: case DECIMAL: case BOOLEAN: - default: return null; } object.type = Type.TIMESTAMP; diff --git a/src/test/java/com/yahoo/validatar/common/TypeSystemTest.java b/src/test/java/com/yahoo/validatar/common/TypeSystemTest.java index ff1904f..5e60b87 100644 --- a/src/test/java/com/yahoo/validatar/common/TypeSystemTest.java +++ b/src/test/java/com/yahoo/validatar/common/TypeSystemTest.java @@ -41,6 +41,10 @@ public class TypeSystemTest { private TypeSystem system = new TypeSystem(); + + private class CustomOperations implements TypeSystem.Operations { + } + public static final double EPSILON = 0.00001; private boolean boolify(TypedObject type) { @@ -662,4 +666,20 @@ public void testFailLogicalAndTimestamp() { public void testFailLogicalAndString() { logicalAnd(asTypedObject("false"), asTypedObject("true")); } + + /************************************/ + + @Test + public void testDispatchedCasting() { + TypedObject first = asTypedObject("42"); + TypedObject result = TypeSystem.perform(TypeSystem.UnaryOperation.CAST, first); + Assert.assertTrue(boolify(isEqualTo(first, result))); + } + + @Test + public void testDefaultCasting() { + TypeSystem.Operations operations = new CustomOperations(); + Assert.assertNull(operations.cast(asTypedObject(42L))); + } } + From 2b9861e67a1b98a12d56cfe291e3df2c0ff24a67 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 00:42:08 -0700 Subject: [PATCH 16/27] Documentation first changes --- README.md | 9 +++++++-- pom.xml | 8 ++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c03d4ce..630a3ba 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # Validatar -[![Build Status](https://travis-ci.org/yahoo/validatar.svg?branch=master)](https://travis-ci.org/yahoo/validatar) [![Coverage Status](https://coveralls.io/repos/yahoo/validatar/badge.svg?branch=master)](https://coveralls.io/r/yahoo/validatar?branch=master) [![Download](https://api.bintray.com/packages/yahoo/maven/validatar/images/download.svg)](https://bintray.com/yahoo/maven/validatar/_latestVersion) +[![Build Status](https://travis-ci.org/yahoo/validatar.svg?branch=master)](https://travis-ci.org/yahoo/validatar) [![Download](https://api.bintray.com/packages/yahoo/maven/validatar/images/download.svg)](https://bintray.com/yahoo/maven/validatar/_latestVersion) + -Functional testing framework for Big Data pipelines. Current support is only for Hive, but we are planning support for Pig as well as others. +Functional testing framework for Big Data pipelines. Currently support querying pipeline results through Hive (HiveServer2) and Pig (PigServer). ## How to build Validatar @@ -82,6 +86,7 @@ Version | Notes 0.1.8 | Null types in Hive results fix 0.1.9 | Empty results handling bug fix 0.2.0 | Internal switch to Java 8. hive-queue is no longer a setting. Use hive-setting. +0.3.0 | Pig support added. ## Members diff --git a/pom.xml b/pom.xml index 3789f8f..83c2d2d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.yahoo.validatar validatar jar - 0.2.1-SNAPSHOT + 0.3.0-SNAPSHOT validatar @@ -160,7 +160,6 @@ com/yahoo/validatar/assertion/Grammar*.java - false @@ -180,13 +179,14 @@ - + org.eluder.coveralls coveralls-maven-plugin 4.0.0 - + + From 6d5df56dc1a0e417f35491aa482a11edd16c6f38 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 00:44:01 -0700 Subject: [PATCH 17/27] Documentation first changes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 630a3ba..8383de7 100644 --- a/README.md +++ b/README.md @@ -36,14 +36,14 @@ name: Test family name : String description: Test family description : String queries: - name: Query name : String : Ex "Analytics" - engine: Execution engine : String ("Hive") + engine: Execution engine : String Ex "hive" or "pig" value: Query : String : Ex "SELECT COUNT(*) AS pv_count FROM page_data" ... tests: - name: Test name : String description: Test description : String asserts: - - Assertion on some query. Query name is prefixed to the value. : Ex: Analytics.pv_count > 10000 + - Assertion on some query. Query name is prefixed to the value. : Ex Analytics.pv_count > 10000 ... ``` From 97c4a2a4b667e33f353704f1c93ce3bc4e73ea85 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 09:13:01 -0700 Subject: [PATCH 18/27] Fixing checkstyle. Adding jacoco. Binding to various mvn lifecycles. --- .travis.yml | 5 +- Makefile | 2 +- README.md | 2 +- pom.xml | 86 +++++++++++++++---- .../yahoo/validatar/common/TypeSystem.java | 85 ++++++++++++++++-- .../yahoo/validatar/parse/ParseManager.java | 1 - .../validatar/report/FormatManagerTest.java | 2 +- 7 files changed, 153 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22e7aa4..858d5d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,11 @@ language: java jdk: - oraclejdk8 +script: +- mvn verify + after_success: -- mvn clean clover2:setup test clover2:aggregate clover2:clover coveralls:report +- mvn coveralls:report - test "${TRAVIS_PULL_REQUEST}" == "false" && test "${TRAVIS_TAG}" != "" && mvn deploy --settings travis/settings.xml cache: diff --git a/Makefile b/Makefile index 203de03..d74c578 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ clean: mvn clean test: - mvn clean checkstyle:check test + mvn clean verify jar: mvn clean package diff --git a/README.md b/README.md index 8383de7..f028f98 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Queries are named, this name is used as a namespace for all the values returned ### Assertions -Assertions are quite flexibile, allowing for the following operations: +Assertions are quite flexible, allowing for the following operations: ``` > : greater than diff --git a/pom.xml b/pom.xml index 83c2d2d..5422aae 100644 --- a/pom.xml +++ b/pom.xml @@ -139,10 +139,20 @@ org.apache.maven.plugins maven-checkstyle-plugin 2.16 - - true - src/main/resources/checkstyle.xml - + + + checkstyle-check + verify + + check + + + com/yahoo/validatar/assertion/Grammar* + true + src/main/resources/checkstyle.xml + + + com.puppycrawl.tools @@ -157,27 +167,65 @@ maven-clover2-plugin 4.0.5 + ${user.home}/.m2/clover.license com/yahoo/validatar/assertion/Grammar*.java - + diff --git a/src/main/java/com/yahoo/validatar/common/TypeSystem.java b/src/main/java/com/yahoo/validatar/common/TypeSystem.java index bdb5f2f..de001a2 100644 --- a/src/main/java/com/yahoo/validatar/common/TypeSystem.java +++ b/src/main/java/com/yahoo/validatar/common/TypeSystem.java @@ -62,45 +62,112 @@ public enum UnaryOperation { * Timestamp to and from Long will do a millis since epoch */ public interface Operations { + /** + * Adds two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject add(TypedObject first, TypedObject second) { return null; } + /** + * Subtracts two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject subtract(TypedObject first, TypedObject second) { return null; } + /** + * Multiplies two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject multiply(TypedObject first, TypedObject second) { return null; } + /** + * Divides two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject divide(TypedObject first, TypedObject second) { return null; } + /** + * Finds the integer remainder after division two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject modulus(TypedObject first, TypedObject second) { return null; } + /** + * Logical ors two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject or(TypedObject first, TypedObject second) { return null; } + /** + * Logical ands two TypedObjects. + * + * @param first The first object. + * @param second The second object. + * @return The result object. + */ default TypedObject and(TypedObject first, TypedObject second) { return null; } + /** + * Logical negates a TypedObject. + * + * @param object The object. + * @return The result object. + */ default TypedObject negate(TypedObject object) { return null; } + /** + * Casts a TypedObject into its given type. + * + * @param object The object. + * @return The result object. + */ default TypedObject cast(TypedObject object) { return null; } - default BinaryOperator dispatch(BinaryOperation operator) { - Objects.requireNonNull(operator); - switch (operator) { + /** + * Given a BinaryOperation, finds the operator for it. Null if it cannot. + * + * @param operation The operation + * @return The result binary operator that can be applied. + */ + default BinaryOperator dispatch(BinaryOperation operation) { + Objects.requireNonNull(operation); + switch (operation) { case ADD: return this::add; case SUBTRACT: @@ -120,9 +187,15 @@ default BinaryOperator dispatch(BinaryOperation operator) { } } - default UnaryOperator dispatch(UnaryOperation operator) { - Objects.requireNonNull(operator); - switch (operator) { + /** + * Given a UnaryOperation, finds the operator for it. Null if it cannot. + * + * @param operation The operation. + * @return The result unary operator that can be applied. + */ + default UnaryOperator dispatch(UnaryOperation operation) { + Objects.requireNonNull(operation); + switch (operation) { case NEGATE: return this::negate; case CAST: diff --git a/src/main/java/com/yahoo/validatar/parse/ParseManager.java b/src/main/java/com/yahoo/validatar/parse/ParseManager.java index e3ffd54..e305ecc 100644 --- a/src/main/java/com/yahoo/validatar/parse/ParseManager.java +++ b/src/main/java/com/yahoo/validatar/parse/ParseManager.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.util.Collection; import java.util.HashMap; import java.util.List; diff --git a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java index 8be5c3a..b53b200 100644 --- a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java +++ b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java @@ -78,7 +78,7 @@ public String getName() { } // Used for tests public static class IllegalAccessFormatter implements Formatter { - public IllegalAccessFormatter() throws IllegalAccessException{ + public IllegalAccessFormatter() throws IllegalAccessException { throw new IllegalAccessException(); } From 92aeaf067c38b67e24b49778de5078d0aadd7394 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 09:17:26 -0700 Subject: [PATCH 19/27] Removing install as it just wastes time --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 858d5d4..a9cbe46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: java jdk: - oraclejdk8 +install: true + script: - mvn verify From ca0c1acfb9714238649d23acd76c088ae53b58ae Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 09:26:16 -0700 Subject: [PATCH 20/27] full is just release artifacts --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d74c578..90c7e26 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: full full: - mvn clean checkstyle:check javadoc:jar package + mvn clean javadoc:jar package clean: mvn clean From cb61492ce34e9f8fcc54c5a0349bce3b175453a6 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 11:39:00 -0700 Subject: [PATCH 21/27] Updating readme --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f028f98..5adc6e8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ # Validatar -[![Build Status](https://travis-ci.org/yahoo/validatar.svg?branch=master)](https://travis-ci.org/yahoo/validatar) [![Download](https://api.bintray.com/packages/yahoo/maven/validatar/images/download.svg)](https://bintray.com/yahoo/maven/validatar/_latestVersion) - +[![Build Status](https://travis-ci.org/yahoo/validatar.svg?branch=master)](https://travis-ci.org/yahoo/validatar) [![Coverage Status](https://coveralls.io/repos/yahoo/validatar/badge.svg?branch=master)](https://coveralls.io/r/yahoo/validatar?branch=master) [![Download](https://api.bintray.com/packages/yahoo/maven/validatar/images/download.svg)](https://bintray.com/yahoo/maven/validatar/_latestVersion) Functional testing framework for Big Data pipelines. Currently support querying pipeline results through Hive (HiveServer2) and Pig (PigServer). @@ -18,13 +14,24 @@ Run: ## How to run -To run Validatar: +To run Hive tests Validatar: export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml +To run Pig tests in Validatar: + + Coming soon! + + You will also need the settings specified for the engine you are planning to run. +Hive needs the JDBC uri of HiveServer2: + ``` + --hive-jdbc "jdbc:hive2:///;> + ``` + + ## Writing Tests ### Test file format @@ -90,7 +97,7 @@ Version | Notes ## Members -Akshai Sarma, akshaisarma@gmail.com +Akshai Sarma, akshaisarma@gmail.com Josh Walters, josh@joshwalters.com ## Contributors From 966d3318bd2aafc25666e026986d129aab14ae6c Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Fri, 18 Sep 2015 11:41:35 -0700 Subject: [PATCH 22/27] Updating readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5adc6e8..bc76f5f 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,9 @@ To run Pig tests in Validatar: You will also need the settings specified for the engine you are planning to run. -Hive needs the JDBC uri of HiveServer2: +Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... ``` - --hive-jdbc "jdbc:hive2:///;> + --hive-jdbc "jdbc:hive2:///; etc> ``` From 0d3918981393dca337b453fc9c8e252590b1722e Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Sat, 19 Sep 2015 13:46:39 -0700 Subject: [PATCH 23/27] Coverage reached --- pom.xml | 4 +- .../yahoo/validatar/common/TypeSystem.java | 1 + .../validatar/execution/hive/Apiary.java | 3 + .../java/com/yahoo/validatar/AppTest.java | 66 +++++++++++++++---- .../com/yahoo/validatar/OutputCaptor.java | 12 +++- .../execution/EngineManagerTest.java | 53 ++++++++++++++- .../validatar/execution/hive/ApiaryTest.java | 49 +++++--------- .../validatar/report/FormatManagerTest.java | 22 ++----- 8 files changed, 145 insertions(+), 65 deletions(-) diff --git a/pom.xml b/pom.xml index 5422aae..28478ac 100644 --- a/pom.xml +++ b/pom.xml @@ -204,12 +204,12 @@ LINE COVEREDRATIO - 0.95 + 0.99 BRANCH COVEREDRATIO - 0.90 + 0.95 CLASS diff --git a/src/main/java/com/yahoo/validatar/common/TypeSystem.java b/src/main/java/com/yahoo/validatar/common/TypeSystem.java index de001a2..df5aaa3 100644 --- a/src/main/java/com/yahoo/validatar/common/TypeSystem.java +++ b/src/main/java/com/yahoo/validatar/common/TypeSystem.java @@ -166,6 +166,7 @@ default TypedObject cast(TypedObject object) { * @return The result binary operator that can be applied. */ default BinaryOperator dispatch(BinaryOperation operation) { + // Can assign to a return value and return it, getting rid of the unreachable default... Objects.requireNonNull(operation); switch (operation) { case ADD: diff --git a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java index 8ee4344..0c63b49 100644 --- a/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java +++ b/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java @@ -179,6 +179,9 @@ TypedObject getAsTypedObject(ResultSet results, int index, int type) throws SQLE case (Types.TIMESTAMP): toReturn = TypeSystem.asTypedObject(results.getTimestamp(index)); break; + case (Types.NULL): + toReturn = null; + break; default: throw new UnsupportedOperationException("Unknown SQL type encountered from Hive: " + type); } diff --git a/src/test/java/com/yahoo/validatar/AppTest.java b/src/test/java/com/yahoo/validatar/AppTest.java index d0a2d31..7411d8b 100644 --- a/src/test/java/com/yahoo/validatar/AppTest.java +++ b/src/test/java/com/yahoo/validatar/AppTest.java @@ -26,15 +26,14 @@ import org.testng.annotations.Test; import java.io.File; -import java.io.FileDescriptor; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.sql.Connection; import java.sql.DriverManager; import java.util.HashMap; import java.util.Map; +import static com.yahoo.validatar.OutputCaptor.redirectToDevNull; +import static com.yahoo.validatar.OutputCaptor.redirectToStandard; import static java.util.Collections.singletonList; public class AppTest { @@ -87,29 +86,31 @@ public CustomEngineManager(String[] arguments) { } @Test - public void testRunTests() throws Exception { - String[] args = {"--report-file", "target/AppTest-testRunTests.xml", + public void testFailRunTests() throws Exception { + String[] args = {"--report-file", "target/AppTest-testFailRunTests.xml", "--hive-driver", "org.h2.Driver", "--hive-jdbc", "jdbc:h2:mem:"}; Map parameterMap = new HashMap<>(); - File emptyTest = new File("src/test/resources/sample-tests/empty-test.yaml"); + File emptyTest = new File("src/test/resources/pig-tests/sample.yaml"); ParseManager parseManager = new ParseManager(); EngineManager engineManager = new CustomEngineManager(args); FormatManager formatManager = new FormatManager(args); App.run(emptyTest, parameterMap, parseManager, engineManager, formatManager); + Assert.assertFalse(new File("target/AppTest-testFailRunTests.xml").exists()); + } + + @Test(expectedExceptions = {RuntimeException.class}) + public void testParameterMissingToken() throws IOException { + String[] args = {"--test-suite", "tests.yaml", "--parameter", "DATE:20140807"}; + OptionSet options = App.parse(args); + App.splitParameters(options, "parameter"); } @Test public void testParameterParsingFailure() throws IOException { - System.setOut(new PrintStream(new FileOutputStream("target/out"))); - System.setErr(new PrintStream(new FileOutputStream("target/err"))); - String[] args = {"--parameter", "DATE:20140807"}; Assert.assertNull(App.parse(args)); - - System.setOut(new PrintStream(new FileOutputStream(FileDescriptor.out))); - System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.err))); } @Test @@ -120,7 +121,7 @@ public void testSimpleParameterParse() throws IOException { "--parameter", "NAME=ALPHA"}; // Parse CLI args - Map paramMap = null; + Map paramMap; OptionSet options = App.parse(args); paramMap = App.splitParameters(options, "parameter"); @@ -131,4 +132,43 @@ public void testSimpleParameterParse() throws IOException { Assert.assertEquals(paramMap.get("DATE"), "2014071800"); Assert.assertEquals(paramMap.get("NAME"), "ALPHA"); } + + @Test + public void testRunTests() throws Exception { + String[] args = {"--report-file", "target/AppTest-testRunTests.xml", + "--hive-driver", "org.h2.Driver", + "--hive-jdbc", "jdbc:h2:mem:"}; + Map parameterMap = new HashMap<>(); + File emptyTest = new File("src/test/resources/sample-tests/empty-test.yaml"); + ParseManager parseManager = new ParseManager(); + EngineManager engineManager = new CustomEngineManager(args); + FormatManager formatManager = new FormatManager(args); + + App.run(emptyTest, parameterMap, parseManager, engineManager, formatManager); + Assert.assertTrue(new File("target/AppTest-testRunTests.xml").exists()); + } + + @Test + public void testMain() throws IOException { + redirectToDevNull(); + + App.main(new String[0]); + + String[] abbreviated = {"--h", "--test-suite", "src/test/resources/sample-tests"}; + App.main(abbreviated); + Assert.assertFalse(new File("target/AppTest-testMainHelpPrinting.xml").exists()); + + String[] nonAbbreviated = {"--help", "--test-suite", "src/test/resources/sample-tests"}; + App.main(nonAbbreviated); + Assert.assertFalse(new File("target/AppTest-testMainHelpPrinting.xml").exists()); + + String[] args = {"--report-file", "target/AppTest-testMainHelpPrinting.xml", + "--test-suite", "src/test/resources/sample-tests", + "--hive-driver", "org.h2.Driver", + "--hive-jdbc", "jdbc:h2:mem:"}; + App.main(args); + Assert.assertFalse(new File("target/AppTest-testMainHelpPrinting.xml").exists()); + + redirectToStandard(); + } } diff --git a/src/test/java/com/yahoo/validatar/OutputCaptor.java b/src/test/java/com/yahoo/validatar/OutputCaptor.java index ce6fe94..5a1027d 100644 --- a/src/test/java/com/yahoo/validatar/OutputCaptor.java +++ b/src/test/java/com/yahoo/validatar/OutputCaptor.java @@ -104,12 +104,20 @@ protected void teardownMockedAppender() { Logger.getRootLogger().addAppender(originalAppender); } - public static void runWithoutOutput(Runnable function) { + public static void redirectToDevNull() { System.setOut(NULL); System.setErr(NULL); - function.run(); + } + + public static void redirectToStandard() { System.setOut(OUT); System.setErr(ERR); } + + public static void runWithoutOutput(Runnable function) { + redirectToDevNull(); + function.run(); + redirectToStandard(); + } } diff --git a/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java b/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java index 96755d1..c77a088 100644 --- a/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java +++ b/src/test/java/com/yahoo/validatar/execution/EngineManagerTest.java @@ -59,9 +59,11 @@ public String getName() { private class MockPassingEngine implements Engine { public static final String ENGINE_NAME = "PASSER"; public boolean helpPrinted = false; + public int timesStarted = 0; @Override public boolean setup(String[] arguments) { + timesStarted++; return true; } @@ -108,6 +110,30 @@ public String getName() { } } + public static class IllegalAccessEngine implements Engine { + public IllegalAccessEngine() throws IllegalAccessException { + throw new IllegalAccessException(); + } + + @Override + public boolean setup(String[] arguments) { + return true; + } + + @Override + public void execute(Query query) { + } + + @Override + public String getName() { + return "IllegalAccess"; + } + + @Override + public void printHelp() { + } + } + List queries; List engines; EngineManager manager; @@ -115,6 +141,7 @@ public String getName() { @BeforeMethod public void setup() { + setupMockedAppender(); query = new Query(); query.engine = MockPassingEngine.ENGINE_NAME; queries = new ArrayList<>(); @@ -125,7 +152,6 @@ public void setup() { engines.add(new MockRunningEngine()); String[] args = {}; manager = new EngineManager(args); - setupMockedAppender(); } @AfterMethod @@ -149,6 +175,12 @@ public void testEngineNotPresent() { Assert.assertTrue(isStringInLog("Engine FAKE_ENGINE not loaded but required by query")); } + @Test + public void testEngineIllegalAccess() { + Assert.assertTrue(isStringInLog("Illegal access while loading class com.yahoo.validatar" + + ".execution.EngineManagerTest$IllegalAccessEngine engine.")); + } + @Test public void testEngineNullQueriesNotNull() { query.engine = null; @@ -171,6 +203,25 @@ public void testPrintHelp() { Assert.assertTrue(engine.helpPrinted); } + @Test + public void testNotStartingEngineIfStarted() { + query.engine = MockPassingEngine.ENGINE_NAME; + manager.setEngines(engines); + manager.startEngines(queries); + manager.startEngines(queries); + MockPassingEngine startedEngine = (MockPassingEngine) engines.stream() + .filter(e -> MockPassingEngine.ENGINE_NAME.equals(e.getName())) + .findFirst().get(); + Assert.assertEquals(startedEngine.timesStarted, 1); + } + + @Test + public void testFailRun() { + query.engine = MockFailingEngine.ENGINE_NAME; + manager.setEngines(engines); + Assert.assertFalse(manager.run(queries)); + } + @Test public void testNormalNoResults() { query.engine = MockPassingEngine.ENGINE_NAME; diff --git a/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java b/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java index 9d7dec4..c3e65a2 100644 --- a/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java +++ b/src/test/java/com/yahoo/validatar/execution/hive/ApiaryTest.java @@ -31,6 +31,7 @@ import java.sql.Timestamp; import java.sql.Types; +import static com.yahoo.validatar.OutputCaptor.runWithoutOutput; import static java.util.Collections.singletonList; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; @@ -55,50 +56,40 @@ public void testGetJDBCConnector() throws Exception { Assert.assertTrue(apiary.setup(args)); Query query = new Query(); query.name = "Test"; - query.value = "SELECT 1 as ONE"; + query.value = "SELECT 1 as ONE, null as TWO"; apiary.execute(query); Assert.assertFalse(query.failed()); Assert.assertEquals((Long) query.getResult().getColumns().get("Test.ONE").get(0).data, (Long) new TypedObject(1L, TypeSystem.Type.LONG).data); + Assert.assertNull(query.getResult().getColumns().get("Test.TWO").get(0)); } @Test - public void testFailSetup() { + public void testFailSetup() throws SQLException, ClassNotFoundException { Apiary apiary = spy(new Apiary()); - try { - doThrow(new SQLException()).when(apiary).setHiveSettings(any(OptionSet.class), any(Statement.class)); - } catch (SQLException se) { - Assert.fail("Should not have thrown an exception"); - } + doThrow(new SQLException()).when(apiary).setHiveSettings(any(OptionSet.class), any(Statement.class)); Assert.assertFalse(apiary.setup(args)); - try { - doThrow(new ClassNotFoundException()).when(apiary).setupConnection(any(OptionSet.class)); - } catch (ClassNotFoundException | SQLException e) { - Assert.fail("Should not have thrown an exception"); - } + doThrow(new ClassNotFoundException()).when(apiary).setupConnection(any(OptionSet.class)); Assert.assertFalse(apiary.setup(args)); } @Test - public void testFailExecution() { + public void testFailExecution() throws Exception { Apiary apiary = spy(new Apiary()); - try { - Statement mocked = mock(Statement.class); - doThrow(new SQLException()).when(mocked).executeQuery(anyString()); - doNothing().when(apiary).setHiveSettings(any(OptionSet.class), any(Statement.class)); - doReturn(mocked).when(apiary).setupConnection(any(OptionSet.class)); - apiary.setup(args); - } catch (ClassNotFoundException | SQLException e) { - Assert.fail("Should not have thrown an exception"); - } + Statement mocked = mock(Statement.class); + doThrow(new SQLException()).when(mocked).executeQuery(anyString()); + doNothing().when(apiary).setHiveSettings(any(OptionSet.class), any(Statement.class)); + doReturn(mocked).when(apiary).setupConnection(any(OptionSet.class)); + apiary.setup(args); Query query = new Query(); apiary.execute(query); Assert.assertTrue(query.failed()); } @Test - public void testHiveSettings() { + public void testHiveSettings() throws SQLException { Apiary apiary = new Apiary(); + runWithoutOutput(() -> apiary.printHelp()); Statement mocked = mock(Statement.class); OptionParser parser = new OptionParser() { { @@ -109,14 +100,10 @@ public void testHiveSettings() { String[] args = {"--hive-setting", "mapreduce.job.queuename=default", "--hive-setting", "hive.execution.engine=tez", "--hive-setting", "hive.execution.engine=mr"}; - try { - apiary.setHiveSettings(parser.parse(args), mocked); - verify(mocked).executeUpdate("set mapreduce.job.queuename=default"); - verify(mocked).executeUpdate("set hive.execution.engine=tez"); - verify(mocked).executeUpdate("set hive.execution.engine=mr"); - } catch (SQLException se) { - Assert.fail("Should not have thrown an exception"); - } + apiary.setHiveSettings(parser.parse(args), mocked); + verify(mocked).executeUpdate("set mapreduce.job.queuename=default"); + verify(mocked).executeUpdate("set hive.execution.engine=tez"); + verify(mocked).executeUpdate("set hive.execution.engine=mr"); } @Test diff --git a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java index b53b200..cfa2ce9 100644 --- a/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java +++ b/src/test/java/com/yahoo/validatar/report/FormatManagerTest.java @@ -16,18 +16,16 @@ package com.yahoo.validatar.report; -import com.yahoo.validatar.OutputCaptor; import com.yahoo.validatar.common.TestSuite; import org.testng.Assert; import org.testng.annotations.Test; -import java.io.FileDescriptor; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.util.List; +import static com.yahoo.validatar.OutputCaptor.runWithoutOutput; + public class FormatManagerTest { // Used for tests private class MockFormatter implements Formatter { @@ -101,30 +99,22 @@ public String getName() { } } - @Test + @Test(expectedExceptions = {RuntimeException.class}) public void testConstructorAndFindFormatterExceptionNoFormatterFound() throws FileNotFoundException { - System.setOut(new PrintStream(new FileOutputStream("target/out"))); - System.setErr(new PrintStream(new FileOutputStream("target/err"))); String[] args = {"--report-format", "INVALID"}; - try { - new FormatManager(args); - Assert.fail("Should have thrown an Exception"); - } catch (RuntimeException re) { - } - System.setOut(new PrintStream(new FileOutputStream(FileDescriptor.out))); - System.setErr(new PrintStream(new FileOutputStream(FileDescriptor.err))); + runWithoutOutput(() -> new FormatManager(args)); } @Test(expectedExceptions = {RuntimeException.class}) public void testFailFormatterSetup() throws IOException { String[] args = {"--report-format", "FailingFormat"}; - FormatManager manager = new FormatManager(args); + new FormatManager(args); } @Test(expectedExceptions = {RuntimeException.class}) public void testFailInstantiation() throws IOException { String[] args = {"--report-format", "IllegalAccessFormat"}; - OutputCaptor.runWithoutOutput(() -> new FormatManager(args)); + runWithoutOutput(() -> new FormatManager(args)); } @Test From c4a7c8456b7d70df1e79207c3ac26b3fdfa896b5 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Sat, 19 Sep 2015 13:55:15 -0700 Subject: [PATCH 24/27] README cleanup --- README.md | 5 ++--- src/test/java/com/yahoo/validatar/AppTest.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bc76f5f..678cc31 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Run: ## How to run -To run Hive tests Validatar: +To run Hive tests in Validatar: export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml @@ -23,7 +23,6 @@ To run Pig tests in Validatar: Coming soon! - You will also need the settings specified for the engine you are planning to run. Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... @@ -36,7 +35,7 @@ Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not a ### Test file format -Test files are written in the YAML format. The schema is as follows: +Test files are written in the YAML format. See example in src/test/resources/. The schema is as follows: ``` name: Test family name : String diff --git a/src/test/java/com/yahoo/validatar/AppTest.java b/src/test/java/com/yahoo/validatar/AppTest.java index 7411d8b..6bace56 100644 --- a/src/test/java/com/yahoo/validatar/AppTest.java +++ b/src/test/java/com/yahoo/validatar/AppTest.java @@ -110,7 +110,7 @@ public void testParameterMissingToken() throws IOException { @Test public void testParameterParsingFailure() throws IOException { String[] args = {"--parameter", "DATE:20140807"}; - Assert.assertNull(App.parse(args)); + Assert.assertNull(new App().parse(args)); } @Test From d9dcd21be02ec2e1f2eb7535f909f33c02651aa5 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 24 Sep 2015 11:55:45 -0700 Subject: [PATCH 25/27] Adding shading for dom4j. Added examples and how to run for pig. --- README.md | 26 ++- pom.xml | 202 ++++++++++-------- .../validatar/execution/EngineManager.java | 2 +- 3 files changed, 131 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 678cc31..dd31c93 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Functional testing framework for Big Data pipelines. Currently support querying pipeline results through Hive (HiveServer2) and Pig (PigServer). +Validatar is currently compiled against *Hive-0.13* and *Pig-0.14*. Running against an older or newer version may result in issues if interfaces have changed. These are relatively minor from experience and can be fixed with relatively minor fixes to engine code. + ## How to build Validatar You need maven/JDK to build Validatar. @@ -14,22 +16,24 @@ Run: ## How to run -To run Hive tests in Validatar: +Use hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App --help (or -h) for Help - export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" - hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml +### To run Hive tests in Validatar: -To run Pig tests in Validatar: + export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" + hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml --hive-jdbc ... - Coming soon! + Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... + ``` + --hive-jdbc "jdbc:hive2:///; etc> + ``` -You will also need the settings specified for the engine you are planning to run. +### To run Pig tests in Validatar: -Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... - ``` - --hive-jdbc "jdbc:hive2:///; etc> - ``` + export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/pig/lib/*" (Add other jars here depending on your pig exec type or if hive/hcat is used in Pig) + hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml --pig-exec-type mr --pig-setting 'mapreduce.job.acl-view-job=*' ... + Pig parameters are not supported in the pig query. Instead, use our parameter substitution (see below). ## Writing Tests @@ -55,6 +59,8 @@ tests: Queries are named, this name is used as a namespace for all the values returned from the query. In the above example, we created a query named "Analytics". It stores the return value "pv_count". We are then able to use this in our later asserts. +Validatar can run a single test file or a folder of test files. Use the --help option to see more details. + ### Assertions Assertions are quite flexible, allowing for the following operations: diff --git a/pom.xml b/pom.xml index 28478ac..6fd9d28 100644 --- a/pom.xml +++ b/pom.xml @@ -13,8 +13,8 @@ scm:git:ssh://git@github.com/yahoo/validatar.git - HEAD - + HEAD + @@ -111,6 +111,32 @@ + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + + shade + + + false + true + jar-with-dependencies + + + org.dom4j + org.shaded.dom4j + + + + + + + org.apache.maven.plugins maven-compiler-plugin @@ -239,93 +265,93 @@ - - - org.testng - testng - 6.8 - test - - - org.mockito - mockito-all - 1.9.5 - test - - - com.h2database - h2 - 1.4.180 - test - - - commons-io - commons-io - 2.4 - test - - - org.antlr - antlr4-runtime - 4.5 - - - net.sf.jopt-simple - jopt-simple - 4.6 - - - log4j - log4j - 1.2.17 - + + + org.antlr + antlr4-runtime + 4.5 + + + net.sf.jopt-simple + jopt-simple + 4.6 + + + log4j + log4j + 1.2.17 + + + org.yaml + snakeyaml + 1.10 + + + dom4j + dom4j + 1.6.1 + + + org.apache.commons + commons-lang3 + 3.3.2 + + + org.reflections + reflections + 0.9.9-RC1 + - - org.yaml - snakeyaml - 1.10 - - - dom4j - dom4j - 1.6.1 - - - org.apache.commons - commons-lang3 - 3.3.2 - - - org.reflections - reflections - 0.9.9-RC1 - - - org.apache.hadoop - hadoop-common - 2.6.0 - provided - - - org.apache.hadoop - hadoop-client - 2.6.0 - provided - - - joda-time - joda-time - 2.6 - provided - - - org.apache.pig - pig - 0.14.0 - h2 - provided - - + + org.testng + testng + 6.8 + test + + + org.mockito + mockito-all + 1.9.5 + test + + + com.h2database + h2 + 1.4.180 + test + + + commons-io + commons-io + 2.4 + test + + + org.apache.hadoop + hadoop-common + 2.6.0 + provided + + + org.apache.hadoop + hadoop-client + 2.6.0 + provided + + + joda-time + joda-time + 2.6 + provided + + + org.apache.pig + pig + 0.14.0 + h2 + provided + + diff --git a/src/main/java/com/yahoo/validatar/execution/EngineManager.java b/src/main/java/com/yahoo/validatar/execution/EngineManager.java index f7db265..09574c5 100644 --- a/src/main/java/com/yahoo/validatar/execution/EngineManager.java +++ b/src/main/java/com/yahoo/validatar/execution/EngineManager.java @@ -96,7 +96,7 @@ public EngineManager(String[] arguments) { try { engine = engineClass.newInstance(); engines.put(engine.getName(), new WorkingEngine(engine)); - log.debug("Added engine " + engine.getName() + " to list of engines."); + log.info("Added engine " + engine.getName() + " to list of engines."); } catch (InstantiationException e) { log.error("Error instantiating " + engineClass + " engine.", e); } catch (IllegalAccessException e) { From adf6697b6b8ca7f6880a75416385d43159e0c869 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 24 Sep 2015 11:57:49 -0700 Subject: [PATCH 26/27] Fixing readme --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dd31c93..93f1cd4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Functional testing framework for Big Data pipelines. Currently support querying pipeline results through Hive (HiveServer2) and Pig (PigServer). -Validatar is currently compiled against *Hive-0.13* and *Pig-0.14*. Running against an older or newer version may result in issues if interfaces have changed. These are relatively minor from experience and can be fixed with relatively minor fixes to engine code. +Validatar is currently compiled against *Pig-0.14*. Running against an older or newer version may result in issues if interfaces have changed. These are relatively minor from experience and can be fixed with relatively minor fixes to engine code. ## How to build Validatar @@ -23,17 +23,15 @@ Use hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App --hel export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml --hive-jdbc ... - Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... - ``` - --hive-jdbc "jdbc:hive2:///; etc> - ``` +Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... + --hive-jdbc "jdbc:hive2:///; etc> ### To run Pig tests in Validatar: export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/pig/lib/*" (Add other jars here depending on your pig exec type or if hive/hcat is used in Pig) hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml --pig-exec-type mr --pig-setting 'mapreduce.job.acl-view-job=*' ... - Pig parameters are not supported in the pig query. Instead, use our parameter substitution (see below). +Pig parameters are not supported in the pig query. Instead, use our parameter substitution (see below). ## Writing Tests From 81a41a4b789403c2179bf51eb11db68db437c235 Mon Sep 17 00:00:00 2001 From: Akshai Sarma Date: Thu, 24 Sep 2015 11:59:09 -0700 Subject: [PATCH 27/27] Fixing readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 93f1cd4..07a1f3f 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,12 @@ Use hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App --hel export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/path/to/hive/jdbc/lib/jars/*" hadoop jar validatar-jar-with-dependencies.jar com.yahoo.validatar.App -s tests/ --report report.xml --hive-jdbc ... -Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use ... FROM DB.TABLE WHERE ... - --hive-jdbc "jdbc:hive2:///; etc> +Hive needs the JDBC uri of HiveServer2. Note that the DB is in the URI. Do not add it if your queries use +``` +... FROM DB.TABLE WHERE ... + +--hive-jdbc "jdbc:hive2:///; etc> +``` ### To run Pig tests in Validatar: