diff --git a/lib/email/format_test_case.js b/lib/email/format_test_case.js
index f983117..9561555 100644
--- a/lib/email/format_test_case.js
+++ b/lib/email/format_test_case.js
@@ -1,6 +1,8 @@
var util = require( 'util' );
var handlebars = require( 'handlebars' );
+var testSuiteHelpers = require('../../lib/test_suite_helpers');
+
/*
* Returns a string representation of the results of running a single
* test case, suitable for display in an HTML email
@@ -8,7 +10,7 @@ var handlebars = require( 'handlebars' );
function formatTestCase( testCase ){
var id = testCase.id;
var input = JSON.stringify( testCase.in, undefined, 4 );
- var result = testCase.results[testCase.full_url];
+ var result = testSuiteHelpers.getMainResult(testCase);
var status = (result.progress === undefined) ? '' :
util.format( '%s ', result.progress );
diff --git a/lib/email/generate_email_body.js b/lib/email/generate_email_body.js
index 6cc30a2..3ec6d20 100644
--- a/lib/email/generate_email_body.js
+++ b/lib/email/generate_email_body.js
@@ -5,16 +5,22 @@ var handlebars = require( 'handlebars' );
var juice = require( 'juice' );
var formatTestCase = require('./format_test_case');
+var testSuiteHelpers = require('../../lib/test_suite_helpers');
function generateEmailBody(suiteResults, config, testSuites) {
+ function shouldDisplayTestSuiteHelper(testSuite) {
+ return !config.quiet || !testSuiteHelpers.allTestsAsExpected(testSuite);
+ }
+
handlebars.registerHelper( 'json', JSON.stringify );
handlebars.registerHelper( 'testCase', formatTestCase );
+ handlebars.registerHelper( 'shouldDisplayTestSuite', shouldDisplayTestSuiteHelper );
var templateParams = { suiteResults: suiteResults, config: config, testSuites: testSuites };
testSuites.forEach(function(suite) {
suite.tests.forEach(function(testCase) {
- testCase.result = testCase.results[testCase.full_url];
+ testCase.result = testSuiteHelpers.getMainResult(testCase);
});
});
diff --git a/lib/processArguments.js b/lib/processArguments.js
index f9fd15a..258aec2 100644
--- a/lib/processArguments.js
+++ b/lib/processArguments.js
@@ -36,7 +36,9 @@ function setUpCommander() {
);
var filesHelp = 'The specific test-suite files to execute. ' +
- 'If not specified, all files in ./test_cases will be run.';
+ 'If not specified, all files in ./test_cases will be run.';
+
+ var noPassingHelp = 'If specified, details of all passing tests will be omitted from results';
commander
.usage( '[flags] [file(s)]' )
@@ -54,6 +56,11 @@ function setUpCommander() {
'-t, --test-type ',
util.format( 'The type of tests to run, as specified in test-cases\' `type` property.' )
)
+ .option(
+ '-q, --quiet',
+ noPassingHelp,
+ false
+ )
.option( 'files', filesHelp )
.parse( process.argv );
@@ -109,7 +116,8 @@ function getConfig() {
outputGenerator: outputGenerator,
testType: commander.testType,
testSuites: testSuites,
- autocomplete: commander.output === 'autocomplete'
+ autocomplete: commander.output === 'autocomplete',
+ quiet: commander.quiet
};
return config;
diff --git a/lib/test_suite_helpers.js b/lib/test_suite_helpers.js
index 64bf2e6..81f2287 100644
--- a/lib/test_suite_helpers.js
+++ b/lib/test_suite_helpers.js
@@ -14,6 +14,35 @@ function getTestCaseProgress( results, testCase ) {
} // all other cases undefined
}
+/**
+ * A single test case can have many results if it was run through autocomplete mode
+ * where each character is turned into its own test. This returns the "primary"
+ * test result.
+ */
+function getMainResult(testCase) {
+ return testCase.results[testCase.full_url];
+}
+
+/* return true if a test was expected to fail, but it passed */
+function isImprovement(result) {
+ return result.progress ==='improvement';
+}
+
+/* return true if a test was not expected to fail, but it did fail */
+function isRegression(result) {
+ return result.progress === 'regression';
+}
+
+/* return true if every test case in a test suite had the expected result
+ * (no improvements or regressions)
+ */
+function allTestsAsExpected(testSuite) {
+ return testSuite.tests.every(function(testCase) {
+ var result = getMainResult(testCase);
+ return !isImprovement(result) && !isRegression(result);
+ });
+}
+
function getLocations() {
try {
return require( path.resolve(process.cwd() + '/locations.json') );
@@ -24,5 +53,7 @@ function getLocations() {
module.exports = {
getTestCaseProgress: getTestCaseProgress,
- getLocations: getLocations
+ getLocations: getLocations,
+ getMainResult: getMainResult,
+ allTestsAsExpected: allTestsAsExpected
};
diff --git a/output_generators/autocomplete.js b/output_generators/autocomplete.js
index 22ee054..822a109 100644
--- a/output_generators/autocomplete.js
+++ b/output_generators/autocomplete.js
@@ -10,6 +10,7 @@ require( 'colors' );
var _ = require( 'lodash' );
var percentageForDisplay = require('../lib/percentageForDisplay');
+var testSuiteHelpers = require('../lib/test_suite_helpers');
/* get a title for this test case with the following features:
* * contains any extra query parameters (api key and of course text don't count)
@@ -17,7 +18,7 @@ var percentageForDisplay = require('../lib/percentageForDisplay');
* - except passing tests which are kept uncolored to avoid color overload
*/
function getTestCaseTitleString(testCase) {
- var original_result = testCase.results[testCase.full_url];
+ var original_result = testSuiteHelpers.getMainResult(testCase);
var colors = {
pass: 'reset', // avoid color overload by keeping passing tests plainly colored
improvement: 'green',
diff --git a/output_generators/email_static/email.html b/output_generators/email_static/email.html
index 700d734..4852559 100644
--- a/output_generators/email_static/email.html
+++ b/output_generators/email_static/email.html
@@ -75,10 +75,12 @@
{{#each testSuites}}
+ {{#if (shouldDisplayTestSuite this) }}
{{name}}
{{#each tests}}
- {{testCase this}}
{{/each}}
+ {{/if}}
{{/each}}
diff --git a/output_generators/json.js b/output_generators/json.js
index a627fb4..b9f2342 100644
--- a/output_generators/json.js
+++ b/output_generators/json.js
@@ -9,6 +9,8 @@ var fs = require('fs-extra');
var terminal = require('./terminal');
var sanitize_filename = require('sanitize-filename');
+var testSuiteHelpers = require('../lib/test_suite_helpers');
+
// replacer for stringifying testCase to avoid circular structure
function replace(key, value) {
if (key === 'results' || key === 'result') {
@@ -21,13 +23,14 @@ function replace(key, value) {
* Format and print a test result to json file.
*/
function saveFailTestResult( testCase ) {
- if( testCase.result.result === 'fail' && testCase.status === 'pass' ) {
+ var result = testSuiteHelpers.getMainResult(testCase);
+ if( result.result === 'fail' && testCase.status === 'pass' ) {
fs.ensureDirSync('./failures');
var recordFailFile = './failures/' + sanitize_filename(
util.format('%s_%s.json', testCase.id, testCase.in.text));
var recordFail = {
test_case: testCase,
- response: testCase.result.response.body.features
+ response: result.response.body.features
};
fs.writeFileSync(recordFailFile, JSON.stringify(recordFail, replace, 2));
}
@@ -40,7 +43,6 @@ function prettyPrintSuiteResults( suiteResults, config, testSuites ) {
testSuites.forEach(function(suite) {
suite.tests.forEach(function(testCase) {
- testCase.result = testCase.results[testCase.full_url];
saveFailTestResult( testCase );
});
});
diff --git a/output_generators/terminal.js b/output_generators/terminal.js
index df60566..abcdae9 100644
--- a/output_generators/terminal.js
+++ b/output_generators/terminal.js
@@ -10,11 +10,13 @@ require( 'colors' );
var util = require( 'util' );
var percentageForDisplay = require('../lib/percentageForDisplay');
+var testSuiteHelpers = require('../lib/test_suite_helpers');
/**
* Format and print a test result to the terminal.
*/
-function prettyPrintResult( result ){
+function prettyPrintTestCase( testCase, quiet ){
+ var result = testSuiteHelpers.getMainResult(testCase);
var id = result.testCase.id;
delete result.testCase.in.api_key; // don't display API key
@@ -32,14 +34,18 @@ function prettyPrintResult( result ){
var status = (result.progress === undefined) ? '' : result.progress.inverse + ' ';
switch( result.result ){
case 'pass':
- console.log( util.format( ' ✔ %s[%s] "%s"', status, id, testDescription ).green );
+ if (!quiet) {
+ console.log(util.format(' ✔ %s[%s] "%s"', status, id, testDescription).green);
+ }
break;
case 'fail':
var color = (result.progress === 'regression') ? 'red' : 'yellow';
- console.log(
- util.format( ' ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ]
- );
+ if (!quiet || color === 'red') {
+ console.log(
+ util.format( ' ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ]
+ );
+ }
break;
case 'placeholder':
@@ -53,6 +59,15 @@ function prettyPrintResult( result ){
}
}
+/*
+ * Decide whether a test suite should be displayed in output
+ * only tests where an unexpected (regression or improvement) result occured should cause
+ * the test suite to display
+ */
+function shouldDisplayTestSuite(testSuite) {
+ return !testSuiteHelpers.allTestsAsExpected(testSuite);
+}
+
/**
* Format and print all of the results from any number of test-suites.
*/
@@ -60,11 +75,15 @@ function prettyPrintSuiteResults( suiteResults, config, testSuites ){
console.log( 'Tests for:', config.endpoint.url.blue + ' (' + config.endpoint.name.blue + ')' );
testSuites.forEach( function(testSuite) {
- console.log();
- console.log(testSuite.name.blue);
- testSuite.tests.forEach( function(testCase) {
- prettyPrintResult( testCase.results[testCase.full_url] );
- });
+
+ if (!config.quiet || shouldDisplayTestSuite(testSuite)) {
+ console.log();
+ console.log(testSuite.name.blue);
+
+ testSuite.tests.forEach( function(testCase) {
+ prettyPrintTestCase( testCase, config.quiet );
+ });
+ }
});
console.log( '\nAggregate test results'.blue );