Merge pull request #110 from 99designs/phpunit-test-warnings-should-cause-failure

PHPUnit test failures should cause test run to fail

Author: julianseeger

src/ParaTest/Runners/PHPUnit/ExecutableTest.php

@@ -81,6 +81,27 @@ public function getStderr()
         return $this->process->getErrorOutput();
     }
 
+    /**
+     * Return any warnings that are in the test output, or false if there are none
+     * @return mixed
+     */
+    public function getWarnings()
+    {
+        if(!$this->process)
+            return false;
+
+        // PHPUnit has a bug where by it doesn't include warnings in the junit
+        // output, but still fails. This is a hacky, imperfect method for extracting them
+        // see https://github.com/sebastianbergmann/phpunit/issues/1317
+        preg_match_all(
+            '/^\d+\) Warning\n(.+?)$/ms',
+            $this->process->getOutput(),
+            $matches
+        );
+
+        return isset($matches[1]) ? $matches[1] : false;
+    }
+
     /**
      * Stop the process and return it's
      * exit code

src/ParaTest/Runners/PHPUnit/ResultPrinter.php

@@ -73,6 +73,13 @@ class ResultPrinter
      */
     protected $colors;
 
+    /**
+     * Warnings generated by the cases
+     *
+     * @var array
+     */
+    protected $warnings = array();
+
     public function __construct(LogInterpreter $results)
     {
         $this->results = $results;
@@ -142,6 +149,7 @@ public function printResults()
         print $this->getHeader();
         print $this->getErrors();
         print $this->getFailures();
+        print $this->getWarnings();
         print $this->getFooter();
     }
 
@@ -158,6 +166,13 @@ public function printFeedback(ExecutableTest $test)
         $feedbackItems = $reader->getFeedback();
         foreach ($feedbackItems as $item)
             $this->printFeedbackItem($item);
+
+        $warnings = $test->getWarnings();
+        if ($warnings) {
+            $this->addWarnings($warnings);
+            foreach ($warnings as $warning)
+                $this->printFeedbackItem('W');
+        }
     }
 
     /**
@@ -186,6 +201,32 @@ public function getHeader()
         return "\n\n" . $this->timer->resourceUsage() . "\n\n";
     }
 
+    /**
+     * Add an array of warning strings. These cause the test run to be shown
+     * as failed
+     */
+    public function addWarnings(array $warnings)
+    {
+        $this->warnings = array_merge($this->warnings, $warnings);
+    }
+
+    /**
+     * Returns warning messages as a string
+     */
+    public function getWarnings()
+    {
+        return $this->getDefects($this->warnings, 'warning');
+    }
+
+    /**
+     * Whether the test run is successful and has no warnings
+     * @return bool
+     */
+    protected function isSuccessful()
+    {
+        return $this->results->isSuccessful() && count($this->warnings) == 0;
+    }
+
     /**
      * Return the footer information reporting success
      * or failure
@@ -194,7 +235,7 @@ public function getHeader()
      */
     public function getFooter()
     {
-        return $this->results->isSuccessful()
+        return $this->isSuccessful()
                     ? $this->getSuccessFooter()
                     : $this->getFailedFooter();
     }

test/fixtures/warning-tests/HasWarningsTest.php

@@ -0,0 +1,22 @@
+<?php
+
+class HasWarningsTest extends PHPUnit_Framework_TestCase
+{
+    public function testPassingTest()
+    {
+        $this->assertTrue(true);
+    }
+
+    private function testPrivateTest()
+    {
+        $this->assertTrue(true);
+    }
+
+    /**
+     * @dataProvider llamas
+     */
+    private function testMissingDataProvider()
+    {
+        $this->assertTrue(true);
+    }
+}

test/functional/PHPUnitTest.php

@@ -283,6 +283,20 @@ public function testFunctionalModeEachTestCalledOnce()
         $this->assertTestsPassed($proc, 2, 2);
     }
 
+    public function testTestsWithWarningsResultInFailure()
+    {
+        $proc = $this->invokeParatest("warning-tests/HasWarningsTest.php",
+            array('bootstrap' => BOOTSTRAP)
+        );
+
+        $output = $proc->getOutput();
+
+        $this->assertEquals(1, $proc->getExitCode(), "Test should fail with 1");
+        $this->assertRegExp("/There were 2 warnings/", $output);
+        $this->assertRegExp("/FAILURES!\n".
+        "Tests: 1, Assertions: 1, Failures: 0, Errors: 0./", $output);
+    }
+
     public function setsCoveragePhpDataProvider()
     {
         return array(