diff --git a/docs/index.md b/docs/index.md index 2d9177d79e..780386e6fb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -687,6 +687,31 @@ describe('retries', function () { }); ``` +## Repeat Tests + +Tests can also be repeated when they pass. This feature can be used to test for leaks and proper tear-down procedures. In this case a test is considered to be successful only if all the runs are successful. + +This feature does re-run a passed test and its corresponding `beforeEach/afterEach` hooks, but not `before/after` hooks. + +If using both `repeat` and `retries`, the test will be run `repeat` times tolerating up to `retries` failures in total. + +```js +describe('repeat', function () { + // Repeat all tests in this suite 4 times + this.repeats(4); + + beforeEach(function () { + browser.get('http://www.yahoo.com'); + }); + + it('should use proper tear-down', function () { + // Specify this test to only retry up to 2 times + this.repeats(2); + expect($('.foo').isDisplayed()).to.eventually.be.true; + }); +}); +``` + ## Dynamically Generating Tests Given Mocha's use of function expressions to define suites and test cases, it's straightforward to generate your tests dynamically. No special syntax is required — plain ol' JavaScript can be used to achieve functionality similar to "parameterized" tests, which you may have seen in other frameworks. @@ -1777,7 +1802,7 @@ describe('Array', function () { it('should not throw an error', function () { (function () { [1, 2, 3].indexOf(4); - }.should.not.throw()); + }).should.not.throw(); }); it('should return -1', function () { [1, 2, 3].indexOf(4).should.equal(-1); @@ -2152,6 +2177,7 @@ mocha.setup({ forbidPending: true, global: ['MyLib'], retries: 3, + repeats: 1, rootHooks: { beforeEach(done) { ... done();} }, slow: '100', timeout: '2000', diff --git a/example/config/.mocharc.js b/example/config/.mocharc.js index e40bea1b71..ff386363ae 100644 --- a/example/config/.mocharc.js +++ b/example/config/.mocharc.js @@ -36,6 +36,7 @@ module.exports = { 'reporter-option': ['foo=bar', 'baz=quux'], // array, not object require: '@babel/register', retries: 1, + repeats: 1, slow: '75', sort: false, spec: ['test/**/*.spec.js'], // the positional arguments! diff --git a/example/config/.mocharc.yml b/example/config/.mocharc.yml index e132b7a99d..062e15e8f1 100644 --- a/example/config/.mocharc.yml +++ b/example/config/.mocharc.yml @@ -40,6 +40,7 @@ reporter-option: # array, not object - 'baz=quux' require: '@babel/register' retries: 1 +repeats: 1 slow: '75' sort: false spec: diff --git a/lib/cli/run-option-metadata.js b/lib/cli/run-option-metadata.js index 492608fbdd..f18122817d 100644 --- a/lib/cli/run-option-metadata.js +++ b/lib/cli/run-option-metadata.js @@ -49,7 +49,7 @@ const TYPES = (exports.types = { 'sort', 'watch' ], - number: ['retries', 'jobs'], + number: ['retries', 'repeats', 'jobs'], string: [ 'config', 'fgrep', diff --git a/lib/cli/run.js b/lib/cli/run.js index fbbe510e94..f4d60d2fcc 100644 --- a/lib/cli/run.js +++ b/lib/cli/run.js @@ -231,6 +231,10 @@ exports.builder = yargs => description: 'Retry failed tests this many times', group: GROUPS.RULES }, + repeats: { + description: 'Repeat passed tests this many times', + group: GROUPS.RULES + }, slow: { default: defaults.slow, description: 'Specify "slow" test threshold (in milliseconds)', diff --git a/lib/context.js b/lib/context.js index 388d308272..6903996dd0 100644 --- a/lib/context.js +++ b/lib/context.js @@ -84,3 +84,18 @@ Context.prototype.retries = function (n) { this.runnable().retries(n); return this; }; + +/** + * Set or get a number of repeats on passed tests + * + * @private + * @param {number} n + * @return {Context} self + */ +Context.prototype.repeats = function (n) { + if (!arguments.length) { + return this.runnable().repeats(); + } + this.runnable().repeats(n); + return this; +}; diff --git a/lib/hook.js b/lib/hook.js index 862271e735..42fb71e6d0 100644 --- a/lib/hook.js +++ b/lib/hook.js @@ -63,6 +63,7 @@ Hook.prototype.error = function (err) { Hook.prototype.serialize = function serialize() { return { $$currentRetry: this.currentRetry(), + $$currentRepeat: this.currentRepeat(), $$fullTitle: this.fullTitle(), $$isPending: Boolean(this.isPending()), $$titlePath: this.titlePath(), diff --git a/lib/mocha.js b/lib/mocha.js index f93865df7e..0a0da4805c 100644 --- a/lib/mocha.js +++ b/lib/mocha.js @@ -170,6 +170,7 @@ exports.run = function (...args) { * @param {string|constructor} [options.reporter] - Reporter name or constructor. * @param {Object} [options.reporterOption] - Reporter settings object. * @param {number} [options.retries] - Number of times to retry failed tests. + * @param {number} [options.repeat] - Number of times to repeat passed tests. * @param {number} [options.slow] - Slow threshold value. * @param {number|string} [options.timeout] - Timeout threshold value. * @param {string} [options.ui] - Interface name. @@ -207,6 +208,10 @@ function Mocha(options = {}) { this.retries(options.retries); } + if ('repeats' in options) { + this.repeats(options.repeats); + } + [ 'allowUncaught', 'asyncOnly', @@ -763,6 +768,25 @@ Mocha.prototype.retries = function (retry) { return this; }; +/** + * Sets the number of times to repeat passed tests. + * + * @public + * @see [CLI option](../#-repeats-n) + * @see [Repeat Tests](../#repeat-tests) + * @param {number} repeats - Number of times to repeat passed tests. + * @return {Mocha} this + * @chainable + * @example + * + * // Allow any passed test to be repeated multiple times + * mocha.repeats(1); + */ +Mocha.prototype.repeats = function (repeats) { + this.suite.repeats(repeats); + return this; +}; + /** * Sets slowness threshold value. * diff --git a/lib/reporters/json-stream.js b/lib/reporters/json-stream.js index 5926587e45..92ebcae206 100644 --- a/lib/reporters/json-stream.js +++ b/lib/reporters/json-stream.js @@ -85,6 +85,7 @@ function clean(test) { file: test.file, duration: test.duration, currentRetry: test.currentRetry(), + currentRepeat: test.currentRepeat(), speed: test.speed }; } diff --git a/lib/reporters/json.js b/lib/reporters/json.js index 6194d8747d..dce5ba4ee5 100644 --- a/lib/reporters/json.js +++ b/lib/reporters/json.js @@ -115,6 +115,7 @@ function clean(test) { file: test.file, duration: test.duration, currentRetry: test.currentRetry(), + currentRepeat: test.currentRepeat(), speed: test.speed, err: cleanCycles(err) }; diff --git a/lib/runnable.js b/lib/runnable.js index fef4941024..04d472c5d6 100644 --- a/lib/runnable.js +++ b/lib/runnable.js @@ -40,6 +40,7 @@ function Runnable(title, fn) { this._timeout = 2000; this._slow = 75; this._retries = -1; + this._repeats = 1; utils.assignNewMochaID(this); Object.defineProperty(this, 'id', { get() { @@ -60,6 +61,7 @@ utils.inherits(Runnable, EventEmitter); Runnable.prototype.reset = function () { this.timedOut = false; this._currentRetry = 0; + this._currentRepeat = 1; this.pending = false; delete this.state; delete this.err; @@ -182,6 +184,18 @@ Runnable.prototype.retries = function (n) { this._retries = n; }; +/** + * Set or get number of repeats. + * + * @private + */ +Runnable.prototype.repeats = function (n) { + if (!arguments.length) { + return this._repeats; + } + this._repeats = n; +}; + /** * Set or get current retry * @@ -194,6 +208,18 @@ Runnable.prototype.currentRetry = function (n) { this._currentRetry = n; }; +/** + * Set or get current repeat + * + * @private + */ +Runnable.prototype.currentRepeat = function (n) { + if (!arguments.length) { + return this._currentRepeat; + } + this._currentRepeat = n; +}; + /** * Return the full title generated by recursively concatenating the parent's * full title. diff --git a/lib/runner.js b/lib/runner.js index 12807725fb..f65f582a0b 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -814,6 +814,14 @@ Runner.prototype.runTests = function (suite, fn) { self.fail(test, err); } self.emit(constants.EVENT_TEST_END, test); + return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + } else if (test.currentRepeat() < test.repeats()) { + var repeatedTest = test.clone(); + repeatedTest.currentRepeat(test.currentRepeat() + 1); + tests.unshift(repeatedTest); + + self.emit(constants.EVENT_TEST_RETRY, test, null); + return self.hookUp(HOOK_TYPE_AFTER_EACH, next); } diff --git a/lib/suite.js b/lib/suite.js index 43cb7556e1..0ddde60f33 100644 --- a/lib/suite.js +++ b/lib/suite.js @@ -73,6 +73,7 @@ function Suite(title, parentContext, isRoot) { this.root = isRoot === true; this.pending = false; this._retries = -1; + this._repeats = 1; this._beforeEach = []; this._beforeAll = []; this._afterEach = []; @@ -127,6 +128,7 @@ Suite.prototype.clone = function () { suite.root = this.root; suite.timeout(this.timeout()); suite.retries(this.retries()); + suite.repeats(this.repeats()); suite.slow(this.slow()); suite.bail(this.bail()); return suite; @@ -174,6 +176,22 @@ Suite.prototype.retries = function (n) { return this; }; +/** + * Set or get number of times to repeat a passed test. + * + * @private + * @param {number|string} n + * @return {Suite|number} for chaining + */ +Suite.prototype.repeats = function (n) { + if (!arguments.length) { + return this._repeats; + } + debug('repeats %d', n); + this._repeats = parseInt(n, 10) || 0; + return this; +}; + /** * Set or get slow `ms` or short-hand such as "2s". * @@ -230,6 +248,7 @@ Suite.prototype._createHook = function (title, fn) { hook.parent = this; hook.timeout(this.timeout()); hook.retries(this.retries()); + hook.repeats(this.repeats()); hook.slow(this.slow()); hook.ctx = this.ctx; hook.file = this.file; @@ -344,6 +363,7 @@ Suite.prototype.addSuite = function (suite) { suite.root = false; suite.timeout(this.timeout()); suite.retries(this.retries()); + suite.repeats(this.repeats()); suite.slow(this.slow()); suite.bail(this.bail()); this.suites.push(suite); @@ -362,6 +382,7 @@ Suite.prototype.addTest = function (test) { test.parent = this; test.timeout(this.timeout()); test.retries(this.retries()); + test.repeats(this.repeats()); test.slow(this.slow()); test.ctx = this.ctx; this.tests.push(test); diff --git a/lib/test.js b/lib/test.js index 0b8fe18be7..b5c8eda6a7 100644 --- a/lib/test.js +++ b/lib/test.js @@ -73,7 +73,9 @@ Test.prototype.clone = function () { test.timeout(this.timeout()); test.slow(this.slow()); test.retries(this.retries()); + test.repeats(this.repeats()); test.currentRetry(this.currentRetry()); + test.currentRepeat(this.currentRepeat()); test.retriedTest(this.retriedTest() || this); test.globals(this.globals()); test.parent = this.parent; @@ -91,6 +93,7 @@ Test.prototype.clone = function () { Test.prototype.serialize = function serialize() { return { $$currentRetry: this._currentRetry, + $$currentRepeat: this._currentRepeat, $$fullTitle: this.fullTitle(), $$isPending: Boolean(this.pending), $$retriedTest: this._retriedTest || null, diff --git a/test/assertions.js b/test/assertions.js index b6ed7b9cc9..315190b8f5 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -306,6 +306,24 @@ module.exports = { }); } ) + .addAssertion( + ' [not] to have repeated test ', + (expect, result, title) => { + expect(result.tests, '[not] to have an item satisfying', { + title, + currentRepeat: expect.it('to be positive') + }); + } + ) + .addAssertion( + ' [not] to have repeated test ', + (expect, result, title, count) => { + expect(result.tests, '[not] to have an item satisfying', { + title, + currentRepeat: count + }); + } + ) .addAssertion( ' [not] to have failed with (error|errors) ', function (expect, result, ...errors) { diff --git a/test/integration/events.spec.js b/test/integration/events.spec.js index b1ec4b5b04..5f190fee36 100644 --- a/test/integration/events.spec.js +++ b/test/integration/events.spec.js @@ -58,6 +58,25 @@ describe('event order', function () { }); }); + describe('--repeats test case', function () { + it('should assert --repeats event order', function (done) { + runMochaJSON( + 'runner/events-repeats.fixture.js', + ['--repeats', '2'], + function (err, res) { + if (err) { + done(err); + return; + } + expect(res, 'to have passed') + .and('to have failed test count', 0) + .and('to have passed test count', 1); + done(); + } + ); + }); + }); + describe('--delay test case', function () { it('should assert --delay event order', function (done) { runMochaJSON( diff --git a/test/integration/fixtures/options/parallel/repeats.fixture.js b/test/integration/fixtures/options/parallel/repeats.fixture.js new file mode 100644 index 0000000000..26d571c3bf --- /dev/null +++ b/test/integration/fixtures/options/parallel/repeats.fixture.js @@ -0,0 +1,14 @@ +describe('repeats suite', function() { + let calls = 0; + this.repeats(3); + + it('should pass', function() { + + }); + + it('should fail on the second call', function () { + calls++; + console.log(`RUN: ${calls}`); + if (calls > 1) throw new Error(); + }); +}); diff --git a/test/integration/fixtures/options/repeats.fixture.js b/test/integration/fixtures/options/repeats.fixture.js new file mode 100644 index 0000000000..1d7ae07807 --- /dev/null +++ b/test/integration/fixtures/options/repeats.fixture.js @@ -0,0 +1,5 @@ +'use strict'; + +describe('repeats', function () { + it('should pass', () => undefined); +}); diff --git a/test/integration/fixtures/repeats/async.fixture.js b/test/integration/fixtures/repeats/async.fixture.js new file mode 100644 index 0000000000..03a6076fc4 --- /dev/null +++ b/test/integration/fixtures/repeats/async.fixture.js @@ -0,0 +1,30 @@ +'use strict'; + +describe('repeats', function () { + var times = 0; + before(function () { + console.log('before'); + }); + + after(function () { + console.log('after'); + }); + + beforeEach(function () { + console.log('before each', times); + }); + + afterEach(function () { + console.log('after each', times); + }); + + it('should allow override and run appropriate hooks', function (done) { + this.timeout(100); + this.repeats(5); + console.log('TEST', times); + if (times++ > 2) { + return setTimeout(done, 300); + } + setTimeout(done, 50); + }); +}); diff --git a/test/integration/fixtures/repeats/early-fail.fixture.js b/test/integration/fixtures/repeats/early-fail.fixture.js new file mode 100644 index 0000000000..d04dcb1cc8 --- /dev/null +++ b/test/integration/fixtures/repeats/early-fail.fixture.js @@ -0,0 +1,9 @@ +'use strict'; +describe('repeats', function () { + this.repeats(2); + var times = 0; + + it('should fail on the second attempt', function () { + if (times++ > 0) throw new Error; + }); +}); diff --git a/test/integration/fixtures/repeats/hooks.fixture.js b/test/integration/fixtures/repeats/hooks.fixture.js new file mode 100644 index 0000000000..b4fc081aaa --- /dev/null +++ b/test/integration/fixtures/repeats/hooks.fixture.js @@ -0,0 +1,27 @@ +'use strict'; + +describe('retries', function () { + var times = 0; + before(function () { + console.log('before'); + }); + + after(function () { + console.log('after'); + }); + + beforeEach(function () { + console.log('before each', times); + }); + + afterEach(function () { + console.log('after each', times); + }); + + it('should allow override and run appropriate hooks', function () { + this.retries(4); + console.log('TEST', times); + times++; + throw new Error('retry error'); + }); +}); diff --git a/test/integration/fixtures/repeats/nested.fixture.js b/test/integration/fixtures/repeats/nested.fixture.js new file mode 100644 index 0000000000..4c7f0b1bd2 --- /dev/null +++ b/test/integration/fixtures/repeats/nested.fixture.js @@ -0,0 +1,15 @@ +'use strict'; + +describe('repeats', function () { + this.repeats(3); + describe('nested', function () { + let count = 0; + + it('should be executed only once', function () { + this.repeats(1); + count++; + if (count > 1) + throw new Error('repeat error'); + }); + }); +}); diff --git a/test/integration/fixtures/runner/events-repeats.fixture.js b/test/integration/fixtures/runner/events-repeats.fixture.js new file mode 100644 index 0000000000..dcbae8022d --- /dev/null +++ b/test/integration/fixtures/runner/events-repeats.fixture.js @@ -0,0 +1,54 @@ +'use strict'; +var Runner = require('../../../../lib/runner.js'); +var assert = require('assert'); +var constants = Runner.constants; +var EVENT_HOOK_BEGIN = constants.EVENT_HOOK_BEGIN; +var EVENT_HOOK_END = constants.EVENT_HOOK_END; +var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; +var EVENT_RUN_END = constants.EVENT_RUN_END; +var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN; +var EVENT_SUITE_END = constants.EVENT_SUITE_END; +var EVENT_TEST_BEGIN = constants.EVENT_TEST_BEGIN; +var EVENT_TEST_END = constants.EVENT_TEST_END; +var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; + +var emitOrder = [ + EVENT_RUN_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_RETRY, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_PASS, + EVENT_TEST_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_SUITE_END, + EVENT_SUITE_END, + EVENT_RUN_END +]; + +var realEmit = Runner.prototype.emit; +Runner.prototype.emit = function(event, ...args) { + assert.strictEqual(event, emitOrder.shift()); + return realEmit.call(this, event, ...args); +}; + +describe('suite A', function() { + before('before', function() {}); + beforeEach('beforeEach', function() {}); + it('test A', () => undefined); + afterEach('afterEach', function() {}); + after('after', function() {}); +}); diff --git a/test/integration/options/parallel.spec.js b/test/integration/options/parallel.spec.js index 825ef150be..5d34c5b513 100644 --- a/test/integration/options/parallel.spec.js +++ b/test/integration/options/parallel.spec.js @@ -146,6 +146,22 @@ describe('--parallel', function () { }); }); + describe('when used with --repeats', function () { + it('should repeat tests appropriately', async function () { + return expect( + runMochaAsync('options/parallel/repeats*', ['--parallel']), + 'when fulfilled', + 'to satisfy', + expect + .it('to have failed') + .and('to have passed test count', 1) + .and('to have pending test count', 0) + .and('to have failed test count', 1) + .and('to contain output', /RUN: 2/) + ); + }); + }); + describe('when used with --allow-uncaught', function () { it('should bubble up an exception', async function () { return expect( diff --git a/test/integration/options/repeats.spec.js b/test/integration/options/repeats.spec.js new file mode 100644 index 0000000000..142406aafc --- /dev/null +++ b/test/integration/options/repeats.spec.js @@ -0,0 +1,23 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--repeats', function () { + var args = []; + + it('should repeat tests', function (done) { + args = ['--repeats', '3']; + var fixture = path.join('options', 'repeats'); + runMochaJSON(fixture, args, function (err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('not to have pending tests') + .and('to have repeated test', 'should pass', 3); + done(); + }); + }); +}); diff --git a/test/integration/repeats.spec.js b/test/integration/repeats.spec.js new file mode 100644 index 0000000000..4b18ac19a2 --- /dev/null +++ b/test/integration/repeats.spec.js @@ -0,0 +1,141 @@ +'use strict'; + +var assert = require('assert'); +var helpers = require('./helpers'); +var runJSON = helpers.runMochaJSON; +var args = []; +var bang = require('../../lib/reporters/base').symbols.bang; + +describe('repeats', function () { + it('are ran in correct order', function (done) { + helpers.runMocha( + 'repeats/hooks.fixture.js', + ['--reporter', 'dot'], + function (err, res) { + var lines, expected; + + if (err) { + done(err); + return; + } + + lines = res.output + .split(helpers.SPLIT_DOT_REPORTER_REGEXP) + .map(function (line) { + return line.trim(); + }) + .filter(function (line) { + return line.length; + }) + .slice(0, -1); + + expected = [ + 'before', + 'before each 0', + 'TEST 0', + 'after each 1', + 'before each 1', + 'TEST 1', + 'after each 2', + 'before each 2', + 'TEST 2', + 'after each 3', + 'before each 3', + 'TEST 3', + 'after each 4', + 'before each 4', + 'TEST 4', + bang + 'after each 5', + 'after' + ]; + + expected.forEach(function (line, i) { + assert.strictEqual(lines[i], line); + }); + + assert.strictEqual(res.code, 1); + done(); + } + ); + }); + + it('should exit early if test fails', function (done) { + runJSON('repeats/early-fail.fixture.js', args, function (err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have passed test count', 0) + .and('to have failed test count', 1) + .and('to have repeated test', 'should fail on the second attempt', 2); + + done(); + }); + }); + + it('should let test override', function (done) { + runJSON('repeats/nested.fixture.js', args, function (err, res) { + if (err) { + done(err); + return; + } + assert.strictEqual(res.stats.passes, 1); + assert.strictEqual(res.stats.failures, 0); + assert.strictEqual(res.stats.tests, 1); + assert.strictEqual(res.tests[0].currentRepeat, 1); + assert.strictEqual(res.code, 0); + done(); + }); + }); + + it('should not hang w/ async test', function (done) { + this.timeout(2500); + helpers.runMocha( + 'repeats/async.fixture.js', + ['--reporter', 'dot'], + function (err, res) { + var lines, expected; + + if (err) { + done(err); + return; + } + + lines = res.output + .split(helpers.SPLIT_DOT_REPORTER_REGEXP) + .map(function (line) { + return line.trim(); + }) + .filter(function (line) { + return line.length; + }) + .slice(0, -1); + + expected = [ + 'before', + 'before each 0', + 'TEST 0', + 'after each 1', + 'before each 1', + 'TEST 1', + 'after each 2', + 'before each 2', + 'TEST 2', + 'after each 3', + 'before each 3', + 'TEST 3', + bang + 'after each 4', + 'after' + ]; + + expected.forEach(function (line, i) { + assert.strictEqual(lines[i], line); + }); + + assert.strictEqual(res.code, 1); + done(); + } + ); + }); +}); diff --git a/test/reporters/helpers.js b/test/reporters/helpers.js index 7159824c56..ad6d8a10e7 100644 --- a/test/reporters/helpers.js +++ b/test/reporters/helpers.js @@ -166,6 +166,7 @@ function makeExpectedTest( expectedFile, expectedDuration, currentRetry, + currentRepeat, expectedBody ) { return { @@ -178,6 +179,9 @@ function makeExpectedTest( currentRetry: function () { return currentRetry; }, + currentRepeat: function () { + return currentRepeat; + }, slow: function () {} }; } diff --git a/test/reporters/json-stream.spec.js b/test/reporters/json-stream.spec.js index b0b9b81407..f79cf6d648 100644 --- a/test/reporters/json-stream.spec.js +++ b/test/reporters/json-stream.spec.js @@ -21,6 +21,7 @@ describe('JSON Stream reporter', function () { var expectedFile = 'someTest.spec.js'; var expectedDuration = 1000; var currentRetry = 1; + var currentRepeat = 1; var expectedSpeed = 'fast'; var expectedTest = makeExpectedTest( expectedTitle, @@ -28,6 +29,7 @@ describe('JSON Stream reporter', function () { expectedFile, expectedDuration, currentRetry, + currentRepeat, expectedSpeed ); var expectedErrorMessage = 'error message'; @@ -78,6 +80,8 @@ describe('JSON Stream reporter', function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + '}]\n' @@ -113,6 +117,8 @@ describe('JSON Stream reporter', function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + ',"err":' + @@ -151,6 +157,8 @@ describe('JSON Stream reporter', function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + ',"err":' + diff --git a/test/reporters/markdown.spec.js b/test/reporters/markdown.spec.js index c81632a1f3..3b11b77545 100644 --- a/test/reporters/markdown.spec.js +++ b/test/reporters/markdown.spec.js @@ -78,6 +78,7 @@ describe('Markdown reporter', function () { }; var expectedDuration = 1000; var currentRetry = 1; + var currentRepeat = 1; var expectedBody = 'some body'; var expectedTest = { title: expectedTitle, @@ -88,6 +89,9 @@ describe('Markdown reporter', function () { currentRetry: function () { return currentRetry; }, + currentRepeat: function () { + return currentRepeat; + }, slow: noop, body: expectedBody }; diff --git a/test/unit/context.spec.js b/test/unit/context.spec.js index 783219d103..8b7a23d0d3 100644 --- a/test/unit/context.spec.js +++ b/test/unit/context.spec.js @@ -95,4 +95,10 @@ describe('methods', function () { expect(this.retries(), 'to be', -1); }); }); + + describe('repeats', function () { + it('should return the number of repeats', function () { + expect(this.repeats(), 'to be', 1); + }); + }); }); diff --git a/test/unit/mocha.spec.js b/test/unit/mocha.spec.js index f966e0c3e7..ca8e81d968 100644 --- a/test/unit/mocha.spec.js +++ b/test/unit/mocha.spec.js @@ -94,9 +94,11 @@ describe('Mocha', function () { mocha = sinon.createStubInstance(Mocha); mocha.timeout.returnsThis(); mocha.retries.returnsThis(); + mocha.repeats.returnsThis(); sinon.stub(Mocha.prototype, 'timeout').returnsThis(); sinon.stub(Mocha.prototype, 'global').returnsThis(); sinon.stub(Mocha.prototype, 'retries').returnsThis(); + sinon.stub(Mocha.prototype, 'repeats').returnsThis(); sinon.stub(Mocha.prototype, 'rootHooks').returnsThis(); sinon.stub(Mocha.prototype, 'parallelMode').returnsThis(); sinon.stub(Mocha.prototype, 'globalSetup').returnsThis(); @@ -155,6 +157,24 @@ describe('Mocha', function () { }); }); + describe('when `repeats` option is present', function () { + it('should attempt to set repeats`', function () { + // eslint-disable-next-line no-new + new Mocha({repeats: 1}); + expect(Mocha.prototype.repeats, 'to have a call satisfying', [1]).and( + 'was called once' + ); + }); + }); + + describe('when `repeats` option is not present', function () { + it('should not attempt to set repeats', function () { + // eslint-disable-next-line no-new + new Mocha({}); + expect(Mocha.prototype.repeats, 'was not called'); + }); + }); + describe('when `rootHooks` option is truthy', function () { it('shouid attempt to set root hooks', function () { // eslint-disable-next-line no-new diff --git a/test/unit/runnable.spec.js b/test/unit/runnable.spec.js index b4ae296478..5397ce3730 100644 --- a/test/unit/runnable.spec.js +++ b/test/unit/runnable.spec.js @@ -145,6 +145,7 @@ describe('Runnable(title, fn)', function () { run.reset(); expect(run.timedOut, 'to be false'); expect(run._currentRetry, 'to be', 0); + expect(run._currentRepeat, 'to be', 1); expect(run.pending, 'to be false'); expect(run.err, 'to be undefined'); expect(run.state, 'to be undefined'); @@ -217,6 +218,14 @@ describe('Runnable(title, fn)', function () { }); }); + describe('#repeats(n)', function () { + it('should set the number of repeats', function () { + var run = new Runnable(); + run.repeats(3); + expect(run.repeats(), 'to be', 3); + }); + }); + describe('.run(fn)', function () { describe('when .pending', function () { it('should not invoke the callback', function (done) { diff --git a/test/unit/runner.spec.js b/test/unit/runner.spec.js index dd96558017..1d92e44471 100644 --- a/test/unit/runner.spec.js +++ b/test/unit/runner.spec.js @@ -528,6 +528,34 @@ describe('Runner', function () { }); }); + it('should emit "retry" when a repeatable test passes', function (done) { + var repeats = 2; + var runs = 0; + var retries = 0; + + var test = new Test('i do nothing', () => { + runs++; + }); + + suite.repeats(repeats); + suite.addTest(test); + + runner.on(EVENT_TEST_RETRY, function (testClone, testErr) { + retries++; + expect(testClone.currentRepeat(), 'to be', runs); + expect(testErr, 'to be', null); + expect(testClone.title, 'to be', test.title); + }); + + runner.run(function (failures) { + expect(failures, 'to be', 0); + expect(runs, 'to be', repeats); + expect(retries, 'to be', repeats - 1); + + done(); + }); + }); + // karma-mocha is inexplicably doing this with a Hook it('should not throw an exception if something emits EVENT_TEST_END with a non-Test object', function () { expect(function () { diff --git a/test/unit/test.spec.js b/test/unit/test.spec.js index 8bd394cc73..dbca1bfb57 100644 --- a/test/unit/test.spec.js +++ b/test/unit/test.spec.js @@ -17,6 +17,8 @@ describe('Test', function () { this._test._slow = 101; this._test._retries = 3; this._test._currentRetry = 1; + this._test._repeats = 3; + this._test._currentRepeat = 1; this._test._allowedGlobals = ['foo']; this._test.parent = 'foo'; this._test.file = 'bar'; @@ -48,6 +50,14 @@ describe('Test', function () { expect(clone1.clone().retriedTest(), 'to be', this._test); }); + it('should copy the repeats value', function () { + expect(this._test.clone().repeats(), 'to be', 3); + }); + + it('should copy the currentRepeat value', function () { + expect(this._test.clone().currentRepeat(), 'to be', 1); + }); + it('should copy the globals value', function () { expect(this._test.clone().globals(), 'not to be empty'); });