Merge pull request #96 from pelias/quieter-output

Add quiet output flag for terminal and email

lib/email/format_test_case.js

@@ -1,14 +1,16 @@
 var util = require( 'util' );
 var handlebars = require( 'handlebars' );
 
+var testSuiteHelpers = require('../../lib/test_suite_helpers');
+
 /*
  * Returns a string representation of the results of running a single
  * test case, suitable for display in an HTML email
  */
 function formatTestCase( testCase ){
   var id = testCase.id;
   var input = JSON.stringify( testCase.in, undefined, 4 );
-  var result = testCase.results[testCase.full_url];
+  var result = testSuiteHelpers.getMainResult(testCase);
   var status = (result.progress === undefined) ? '' :
     util.format( '<span class="status">%s</span> ', result.progress );
 

lib/email/generate_email_body.js

@@ -5,16 +5,22 @@ var handlebars = require( 'handlebars' );
 var juice = require( 'juice' );
 
 var formatTestCase = require('./format_test_case');
+var testSuiteHelpers = require('../../lib/test_suite_helpers');
 
 function generateEmailBody(suiteResults, config, testSuites) {
+  function shouldDisplayTestSuiteHelper(testSuite) {
+    return !config.quiet || !testSuiteHelpers.allTestsAsExpected(testSuite);
+  }
+
   handlebars.registerHelper( 'json', JSON.stringify );
   handlebars.registerHelper( 'testCase', formatTestCase );
+  handlebars.registerHelper( 'shouldDisplayTestSuite', shouldDisplayTestSuiteHelper );
 
   var templateParams = { suiteResults: suiteResults, config: config, testSuites: testSuites };
 
   testSuites.forEach(function(suite) {
     suite.tests.forEach(function(testCase) {
-      testCase.result = testCase.results[testCase.full_url];
+      testCase.result = testSuiteHelpers.getMainResult(testCase);
     });
   });
 

lib/processArguments.js

@@ -36,7 +36,9 @@ function setUpCommander() {
   );
 
   var filesHelp = 'The specific test-suite files to execute. ' +
-  'If not specified, all files in ./test_cases will be run.';
+    'If not specified, all files in ./test_cases will be run.';
+
+  var noPassingHelp = 'If specified, details of all passing tests will be omitted from results';
 
   commander
   .usage( '[flags] [file(s)]' )
@@ -54,6 +56,11 @@ function setUpCommander() {
     '-t, --test-type <testType>',
     util.format( 'The type of tests to run, as specified in test-cases\' `type` property.' )
   )
+  .option(
+    '-q, --quiet',
+    noPassingHelp,
+    false
+  )
   .option( 'files', filesHelp )
   .parse( process.argv );
 
@@ -109,7 +116,8 @@ function getConfig() {
     outputGenerator: outputGenerator,
     testType: commander.testType,
     testSuites: testSuites,
-    autocomplete: commander.output === 'autocomplete'
+    autocomplete: commander.output === 'autocomplete',
+    quiet: commander.quiet
   };
 
   return config;

lib/test_suite_helpers.js

@@ -14,6 +14,35 @@ function getTestCaseProgress( results, testCase ) {
   } // all other cases undefined
 }
 
+/**
+ * A single test case can have many results if it was run through autocomplete mode
+ * where each character is turned into its own test. This returns the "primary"
+ * test result.
+ */
+function getMainResult(testCase) {
+  return testCase.results[testCase.full_url];
+}
+
+/* return true if a test was expected to fail, but it passed */
+function isImprovement(result) {
+  return result.progress ==='improvement';
+}
+
+/* return true if a test was not expected to fail, but it did fail */
+function isRegression(result) {
+  return result.progress === 'regression';
+}
+
+/* return true if every test case in a test suite had the expected result
+ * (no improvements or regressions)
+ */
+function allTestsAsExpected(testSuite) {
+  return testSuite.tests.every(function(testCase) {
+    var result = getMainResult(testCase);
+    return !isImprovement(result) && !isRegression(result);
+  });
+}
+
 function getLocations() {
   try {
     return require( path.resolve(process.cwd() + '/locations.json') );
@@ -24,5 +53,7 @@ function getLocations() {
 
 module.exports = {
   getTestCaseProgress: getTestCaseProgress,
-  getLocations: getLocations
+  getLocations: getLocations,
+  getMainResult: getMainResult,
+  allTestsAsExpected: allTestsAsExpected
 };

output_generators/autocomplete.js

@@ -10,14 +10,15 @@ require( 'colors' );
 var _ = require( 'lodash' );
 
 var percentageForDisplay = require('../lib/percentageForDisplay');
+var testSuiteHelpers = require('../lib/test_suite_helpers');
 
 /* get a title for this test case with the following features:
  * * contains any extra query parameters (api key and of course text don't count)
  * * changes the color of the text to match the test result of the original query
  *   - except passing tests which are kept uncolored to avoid color overload
  */
 function getTestCaseTitleString(testCase) {
-  var original_result = testCase.results[testCase.full_url];
+  var original_result = testSuiteHelpers.getMainResult(testCase);
   var colors = {
     pass: 'reset', // avoid color overload by keeping passing tests plainly colored
     improvement: 'green',

output_generators/email_static/email.html

@@ -75,10 +75,12 @@
 </div>
 
 {{#each testSuites}}
+  {{#if (shouldDisplayTestSuite this) }}
 	<h2>{{name}}</h2>
 	<ul>
 		{{#each tests}}
 			<li class="{{result.result}}-{{status}}">{{testCase this}}</li>
 		{{/each}}
 	</ul>
+  {{/if}}
 {{/each}}

output_generators/json.js

@@ -9,6 +9,8 @@ var fs = require('fs-extra');
 var terminal = require('./terminal');
 var sanitize_filename = require('sanitize-filename');
 
+var testSuiteHelpers = require('../lib/test_suite_helpers');
+
 // replacer for stringifying testCase to avoid circular structure
 function replace(key, value) {
   if (key === 'results' || key === 'result') {
@@ -21,13 +23,14 @@ function replace(key, value) {
  * Format and print a test result to json file.
  */
 function saveFailTestResult( testCase ) {
-  if( testCase.result.result === 'fail' && testCase.status === 'pass' ) {
+  var result = testSuiteHelpers.getMainResult(testCase);
+  if( result.result === 'fail' && testCase.status === 'pass' ) {
     fs.ensureDirSync('./failures');
     var recordFailFile = './failures/' + sanitize_filename(
         util.format('%s_%s.json', testCase.id, testCase.in.text));
     var recordFail = {
       test_case: testCase,
-      response: testCase.result.response.body.features
+      response: result.response.body.features
     };
     fs.writeFileSync(recordFailFile, JSON.stringify(recordFail, replace, 2));
   }
@@ -40,7 +43,6 @@ function prettyPrintSuiteResults( suiteResults, config, testSuites ) {
 
   testSuites.forEach(function(suite) {
     suite.tests.forEach(function(testCase) {
-      testCase.result = testCase.results[testCase.full_url];
       saveFailTestResult( testCase );
     });
   });

output_generators/terminal.js

@@ -10,11 +10,13 @@ require( 'colors' );
 var util = require( 'util' );
 
 var percentageForDisplay = require('../lib/percentageForDisplay');
+var testSuiteHelpers = require('../lib/test_suite_helpers');
 
 /**
  * Format and print a test result to the terminal.
  */
-function prettyPrintResult( result ){
+function prettyPrintTestCase( testCase, quiet ){
+  var result = testSuiteHelpers.getMainResult(testCase);
   var id = result.testCase.id;
   delete result.testCase.in.api_key; // don't display API key
 
@@ -32,14 +34,18 @@ function prettyPrintResult( result ){
   var status = (result.progress === undefined) ? '' : result.progress.inverse + ' ';
   switch( result.result ){
     case 'pass':
-      console.log( util.format( '  ✔ %s[%s] "%s"', status, id, testDescription ).green );
+      if (!quiet) {
+        console.log(util.format('  ✔ %s[%s] "%s"', status, id, testDescription).green);
+      }
       break;
 
     case 'fail':
       var color = (result.progress === 'regression') ? 'red' : 'yellow';
-      console.log(
-        util.format( '  ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ]
-      );
+      if (!quiet || color === 'red') {
+        console.log(
+          util.format( '  ✘ %s[%s] "%s": %s', status, id, testDescription, result.msg )[ color ]
+        );
+      }
       break;
 
     case 'placeholder':
@@ -53,18 +59,31 @@ function prettyPrintResult( result ){
   }
 }
 
+/*
+ * Decide whether a test suite should be displayed in output
+ * only tests where an unexpected (regression or improvement) result occured should cause
+ * the test suite to display
+ */
+function shouldDisplayTestSuite(testSuite) {
+  return !testSuiteHelpers.allTestsAsExpected(testSuite);
+}
+
 /**
  * Format and print all of the results from any number of test-suites.
  */
 function prettyPrintSuiteResults( suiteResults, config, testSuites ){
   console.log( 'Tests for:', config.endpoint.url.blue + ' (' + config.endpoint.name.blue + ')' );
 
   testSuites.forEach( function(testSuite) {
-    console.log();
-    console.log(testSuite.name.blue);
-    testSuite.tests.forEach( function(testCase) {
-      prettyPrintResult( testCase.results[testCase.full_url] );
-    });
+
+    if (!config.quiet || shouldDisplayTestSuite(testSuite)) {
+      console.log();
+      console.log(testSuite.name.blue);
+
+      testSuite.tests.forEach( function(testCase) {
+        prettyPrintTestCase( testCase, config.quiet );
+      });
+    }
   });
 
   console.log( '\nAggregate test results'.blue );