diff --git a/lib/email/format_test_case.js b/lib/email/format_test_case.js index f983117..9561555 100644 --- a/lib/email/format_test_case.js +++ b/lib/email/format_test_case.js @@ -1,6 +1,8 @@ var util = require( 'util' ); var handlebars = require( 'handlebars' ); +var testSuiteHelpers = require('../../lib/test_suite_helpers'); + /* * Returns a string representation of the results of running a single * test case, suitable for display in an HTML email @@ -8,7 +10,7 @@ var handlebars = require( 'handlebars' ); function formatTestCase( testCase ){ var id = testCase.id; var input = JSON.stringify( testCase.in, undefined, 4 ); - var result = testCase.results[testCase.full_url]; + var result = testSuiteHelpers.getMainResult(testCase); var status = (result.progress === undefined) ? '' : util.format( '%s ', result.progress ); diff --git a/lib/email/generate_email_body.js b/lib/email/generate_email_body.js index 6cc30a2..3ec6d20 100644 --- a/lib/email/generate_email_body.js +++ b/lib/email/generate_email_body.js @@ -5,16 +5,22 @@ var handlebars = require( 'handlebars' ); var juice = require( 'juice' ); var formatTestCase = require('./format_test_case'); +var testSuiteHelpers = require('../../lib/test_suite_helpers'); function generateEmailBody(suiteResults, config, testSuites) { + function shouldDisplayTestSuiteHelper(testSuite) { + return !config.quiet || !testSuiteHelpers.allTestsAsExpected(testSuite); + } + handlebars.registerHelper( 'json', JSON.stringify ); handlebars.registerHelper( 'testCase', formatTestCase ); + handlebars.registerHelper( 'shouldDisplayTestSuite', shouldDisplayTestSuiteHelper ); var templateParams = { suiteResults: suiteResults, config: config, testSuites: testSuites }; testSuites.forEach(function(suite) { suite.tests.forEach(function(testCase) { - testCase.result = testCase.results[testCase.full_url]; + testCase.result = testSuiteHelpers.getMainResult(testCase); }); }); diff --git a/lib/processArguments.js b/lib/processArguments.js index f9fd15a..258aec2 100644 --- a/lib/processArguments.js +++ b/lib/processArguments.js @@ -36,7 +36,9 @@ function setUpCommander() { ); var filesHelp = 'The specific test-suite files to execute. ' + - 'If not specified, all files in ./test_cases will be run.'; + 'If not specified, all files in ./test_cases will be run.'; + + var noPassingHelp = 'If specified, details of all passing tests will be omitted from results'; commander .usage( '[flags] [file(s)]' ) @@ -54,6 +56,11 @@ function setUpCommander() { '-t, --test-type ', util.format( 'The type of tests to run, as specified in test-cases\' `type` property.' ) ) + .option( + '-q, --quiet', + noPassingHelp, + false + ) .option( 'files', filesHelp ) .parse( process.argv ); @@ -109,7 +116,8 @@ function getConfig() { outputGenerator: outputGenerator, testType: commander.testType, testSuites: testSuites, - autocomplete: commander.output === 'autocomplete' + autocomplete: commander.output === 'autocomplete', + quiet: commander.quiet }; return config; diff --git a/lib/test_suite_helpers.js b/lib/test_suite_helpers.js index 64bf2e6..81f2287 100644 --- a/lib/test_suite_helpers.js +++ b/lib/test_suite_helpers.js @@ -14,6 +14,35 @@ function getTestCaseProgress( results, testCase ) { } // all other cases undefined } +/** + * A single test case can have many results if it was run through autocomplete mode + * where each character is turned into its own test. This returns the "primary" + * test result. + */ +function getMainResult(testCase) { + return testCase.results[testCase.full_url]; +} + +/* return true if a test was expected to fail, but it passed */ +function isImprovement(result) { + return result.progress ==='improvement'; +} + +/* return true if a test was not expected to fail, but it did fail */ +function isRegression(result) { + return result.progress === 'regression'; +} + +/* return true if every test case in a test suite had the expected result + * (no improvements or regressions) + */ +function allTestsAsExpected(testSuite) { + return testSuite.tests.every(function(testCase) { + var result = getMainResult(testCase); + return !isImprovement(result) && !isRegression(result); + }); +} + function getLocations() { try { return require( path.resolve(process.cwd() + '/locations.json') ); @@ -24,5 +53,7 @@ function getLocations() { module.exports = { getTestCaseProgress: getTestCaseProgress, - getLocations: getLocations + getLocations: getLocations, + getMainResult: getMainResult, + allTestsAsExpected: allTestsAsExpected }; diff --git a/output_generators/autocomplete.js b/output_generators/autocomplete.js index 22ee054..822a109 100644 --- a/output_generators/autocomplete.js +++ b/output_generators/autocomplete.js @@ -10,6 +10,7 @@ require( 'colors' ); var _ = require( 'lodash' ); var percentageForDisplay = require('../lib/percentageForDisplay'); +var testSuiteHelpers = require('../lib/test_suite_helpers'); /* get a title for this test case with the following features: * * contains any extra query parameters (api key and of course text don't count) @@ -17,7 +18,7 @@ var percentageForDisplay = require('../lib/percentageForDisplay'); * - except passing tests which are kept uncolored to avoid color overload */ function getTestCaseTitleString(testCase) { - var original_result = testCase.results[testCase.full_url]; + var original_result = testSuiteHelpers.getMainResult(testCase); var colors = { pass: 'reset', // avoid color overload by keeping passing tests plainly colored improvement: 'green', diff --git a/output_generators/email_static/email.html b/output_generators/email_static/email.html index 700d734..4852559 100644 --- a/output_generators/email_static/email.html +++ b/output_generators/email_static/email.html @@ -75,10 +75,12 @@ {{#each testSuites}} + {{#if (shouldDisplayTestSuite this) }}

{{name}}

+ {{/if}} {{/each}} diff --git a/output_generators/json.js b/output_generators/json.js index a627fb4..b9f2342 100644 --- a/output_generators/json.js +++ b/output_generators/json.js @@ -9,6 +9,8 @@ var fs = require('fs-extra'); var terminal = require('./terminal'); var sanitize_filename = require('sanitize-filename'); +var testSuiteHelpers = require('../lib/test_suite_helpers'); + // replacer for stringifying testCase to avoid circular structure function replace(key, value) { if (key === 'results' || key === 'result') { @@ -21,13 +23,14 @@ function replace(key, value) { * Format and print a test result to json file. */ function saveFailTestResult( testCase ) { - if( testCase.result.result === 'fail' && testCase.status === 'pass' ) { + var result = testSuiteHelpers.getMainResult(testCase); + if( result.result === 'fail' && testCase.status === 'pass' ) { fs.ensureDirSync('./failures'); var recordFailFile = './failures/' + sanitize_filename( util.format('%s_%s.json', testCase.id, testCase.in.text)); var recordFail = { test_case: testCase, - response: testCase.result.response.body.features + response: result.response.body.features }; fs.writeFileSync(recordFailFile, JSON.stringify(recordFail, replace, 2)); } @@ -40,7 +43,6 @@ function prettyPrintSuiteResults( suiteResults, config, testSuites ) { testSuites.forEach(function(suite) { suite.tests.forEach(function(testCase) { - testCase.result = testCase.results[testCase.full_url]; saveFailTestResult( testCase ); }); }); diff --git a/output_generators/terminal.js b/output_generators/terminal.js index df60566..abcdae9 100644 --- a/output_generators/terminal.js +++ b/output_generators/terminal.js @@ -10,11 +10,13 @@ require( 'colors' ); var util = require( 'util' ); var percentageForDisplay = require('../lib/percentageForDisplay'); +var testSuiteHelpers = require('../lib/test_suite_helpers'); /** * Format and print a test result to the terminal. */ -function prettyPrintResult( result ){ +function prettyPrintTestCase( testCase, quiet ){ + var result = testSuiteHelpers.getMainResult(testCase); var id = result.testCase.id; delete result.testCase.in.api_key; // don't display API key @@ -32,14 +34,18 @@ function prettyPrintResult( result ){ var status = (result.progress === undefined) ? '' : result.progress.inverse + ' '; switch( result.result ){ case 'pass': - console.log( util.format( ' ✔ %s[%s] "%s"', status, id, testDescription ).green ); + if (!quiet) { + console.log(util.format(' ✔ %s[%s] "%s"', status, id, testDescription).green); + } break; case 'fail': var color = (result.progress === 'regression') ? 'red' : 'yellow'; - console.log( - util.format( ' ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ] - ); + if (!quiet || color === 'red') { + console.log( + util.format( ' ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ] + ); + } break; case 'placeholder': @@ -53,6 +59,15 @@ function prettyPrintResult( result ){ } } +/* + * Decide whether a test suite should be displayed in output + * only tests where an unexpected (regression or improvement) result occured should cause + * the test suite to display + */ +function shouldDisplayTestSuite(testSuite) { + return !testSuiteHelpers.allTestsAsExpected(testSuite); +} + /** * Format and print all of the results from any number of test-suites. */ @@ -60,11 +75,15 @@ function prettyPrintSuiteResults( suiteResults, config, testSuites ){ console.log( 'Tests for:', config.endpoint.url.blue + ' (' + config.endpoint.name.blue + ')' ); testSuites.forEach( function(testSuite) { - console.log(); - console.log(testSuite.name.blue); - testSuite.tests.forEach( function(testCase) { - prettyPrintResult( testCase.results[testCase.full_url] ); - }); + + if (!config.quiet || shouldDisplayTestSuite(testSuite)) { + console.log(); + console.log(testSuite.name.blue); + + testSuite.tests.forEach( function(testCase) { + prettyPrintTestCase( testCase, config.quiet ); + }); + } }); console.log( '\nAggregate test results'.blue );