Improve release build and reduce coverage text report to only show un…

…covered lines and branches, along with coverage while running test through valgrind
aiobofh · Aug 18, 2017 · 2da515a · 2da515a
1 parent ccfae0c
commit 2da515a
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 34 deletions.
diff --git a/Makefile b/Makefile
@@ -2,13 +2,13 @@
 # Makefile to build releases of CUTest
 #
 
-export COVERAGE=0
+#export COVERAGE=1
 VERSION=$(shell grep 'CUTEST_VERSION' src/cutest.h | cut -d'"' -f2)
 
 all:
-	@echo "========================" && \
-	echo "Running regression tests" && \
-	echo "========================" && \
+	@echo "======================================" && \
+	echo "Running regression tests with valgrind" && \
+	echo "======================================" && \
 	echo "" && \
 	$(MAKE) -r --no-print-directory regression_tests && \
 	$(MAKE) -r --no-print-directory README.rst && \
@@ -17,11 +17,11 @@ all:
 
 regression_tests:
 	@echo "examples:" && \
-	$(MAKE) -r --no-print-directory -C examples && echo "OK" && \
+	$(MAKE) -r --no-print-directory -C examples valgrind >/dev/null && echo "OK" && \
 	echo "my_project_with_a_test_folder_inside_the_src_folder:" && \
-	$(MAKE) -r --no-print-directory -C examples/complex_directory_structure/my_project_with_a_test_folder_inside_the_src_folder/src && echo "OK" && \
+	$(MAKE) -r --no-print-directory -C examples/complex_directory_structure/my_project_with_a_test_folder_inside_the_src_folder/src valgrind >/dev/null && echo "OK" && \
 	echo "my_project_with_separate_src_and_test_folders:" && \
-	$(MAKE) -r --no-print-directory -C examples/complex_directory_structure/my_project_with_separate_src_and_test_folders && echo "OK" && \
+	$(MAKE) -r --no-print-directory -C examples/complex_directory_structure/my_project_with_separate_src_and_test_folders valgrind >/dev/null && echo "OK" && \
 	echo ""
 
 release: cutest-$(VERSION).tar.gz
@@ -48,9 +48,9 @@ cutest-$(VERSION).tar.gz:
 	echo "OK" && \
 	echo "" && \
 	tar -xzf cutest-$(VERSION).tar.gz && cp Makefile cutest-$(VERSION)/. && \
-	echo "========================" && \
-	echo "Testing tar-ball" && \
-	echo "========================" && \
+	echo "==============================" && \
+	echo "Testing tar-ball with valgrind" && \
+	echo "==============================" && \
 	echo "" && \
 	$(MAKE) -r --no-print-directory -C cutest-$(VERSION) regression_tests && \
 	rm -rf cutest-$(VERSION) && \
@@ -68,7 +68,9 @@ README.rst: examples/cutest_help.rst
 	echo "" && \
 	mv $^ $@ && echo "OK" && \
 	$(MAKE) -r --no-print-directory -C examples clean && \
-	echo ""
+	echo "" && \
+	git add $@
+
 clean:
 	@rm -rf tmp cutest-* *~ && \
 	$(MAKE) -r --no-print-directory -C examples clean && \

diff --git a/README.rst b/README.rst
@@ -32,6 +32,9 @@ Version history
 * v1.0.2 yyyy-mm-dd Release work flow enhancements
 
   - Fixed the documentation generator to be run before release
+  - Made the release build more determenistic and reduced text output
+  - Reduced coverage text report to console to only show non-covered
+  - Enabled coverage reports also when running test through valgrind
 
 * v1.0.1 2017-08-15 Fix-up release
 

diff --git a/examples/complex_directory_structure/my_project_with_separate_src_and_test_folders/Makefile b/examples/complex_directory_structure/my_project_with_separate_src_and_test_folders/Makefile
@@ -28,6 +28,9 @@ src:
 check:
 	@$(MAKE) -r --no-print-directory -C test
 
+valgrind:
+	@$(MAKE) -r --no-print-directory -C test valgrind
+
 clean::
 	@rm -f *~ obj; \
 	$(MAKE) -r --no-print-directory -C src clean; \

diff --git a/src/coverage.mk b/src/coverage.mk
@@ -21,15 +21,27 @@
 CUTEST_CFLAGS+=-fprofile-arcs -ftest-coverage
 CUTEST_COVERAGE_DIR?=.
 
+
 coverage.xml: check
 	$(Q)gcovr -r $(CUTEST_COVERAGE_DIR) -e '/usr.*' -e '.*_test.c' -e 'cutest.h' -e '.*_mocks.h' -e 'error.h' -e '.*_test_run.c' -x > $@
 
+.NOTPARALLEL: lines.cov
+lines.cov: $(subst .c,,$(wildcard $(CUTEST_TEST_DIR)/*_test.c))
+	$(Q)gcovr -r $(CUTEST_COVERAGE_DIR) -e '/usr.*' -e '.*_test.c' -e '.*cutest.h' -e '.*_mocks.h' -e 'error.h' -e '.*_test_run.c' $(CUTEST_COVERAGE_EXCLUDE) | egrep -v '^File' | egrep -v '^-' | egrep -v '^Directory' | grep -v 'GCC Code' | grep -v '100%' | grep -v "\-\-\%" > $@; true
+
+.NOTPARALLEL: branches.cov
+branches.cov: $(subst .c,,$(wildcard $(CUTEST_TEST_DIR)/*_test.c))
+	$(Q)gcovr -r $(CUTEST_COVERAGE_DIR) -e '/usr.*' -e '.*_test.c' -e '.*cutest.h'  -e '.*_mocks.h' -e 'error.h' -e '.*_test_run.c' $(CUTEST_COVERAGE_EXCLUDE) -b | egrep -v '^File' | egrep -v '^-' | egrep -v '^Directory' | grep -v 'GCC Code' | grep -v '100%' | grep -v "\-\-\%" > $@; true
+
+output_coverage: lines.cov branches.cov
+	$(Q)test -s lines.cov && echo "Lines not covered:" >&2 && cat lines.cov >&2 && echo >&2; \
+	test -s branches.cov && echo "Branches not covered:" >&2; cat branches.cov >&2
+
 check::
-	$(Q)echo "Lines covered:"; \
-	gcovr -r $(CUTEST_COVERAGE_DIR) -e '/usr.*' -e '.*_test.c' -e '.*cutest.h' -e '.*_mocks.h' -e 'error.h' -e '.*_test_run.c' $(CUTEST_COVERAGE_EXCLUDE) | egrep -v '^File' | egrep -v '^-' | egrep -v '^Directory' | grep -v 'GCC Code'; \
-	echo ""; \
-	echo "Branches covered:"; \
-	gcovr -r $(CUTEST_COVERAGE_DIR) -e '/usr.*' -e '.*_test.c' -e '.*cutest.h'  -e '.*_mocks.h' -e 'error.h' -e '.*_test_run.c' $(CUTEST_COVERAGE_EXCLUDE) -b | egrep -v '^File' | egrep -v '^-' | egrep -v '^Directory' | grep -v 'GCC Code';
+	$(Q)$(MAKE) -r --no-print-directory output_coverage
+
+valgrind::
+	$(Q)$(MAKE) -r --no-print-directory output_coverage
 
 clean::
-	$(Q)rm -rf coverage.xml *.gcno *.gcda
+	$(Q)rm -rf coverage.xml *.gcno *.gcda *.cov
diff --git a/src/cutest.h b/src/cutest.h
@@ -53,6 +53,9 @@
  * * v1.0.2 yyyy-mm-dd Release work flow enhancements
  *
  *   - Fixed the documentation generator to be run before release
+ *   - Made the release build more determenistic and reduced text output
+ *   - Reduced coverage text report to console to only show non-covered
+ *   - Enabled coverage reports also when running test through valgrind
  *
  * * v1.0.1 2017-08-15 Fix-up release
  *
@@ -630,12 +633,13 @@ int cutest_assert_eq_default(const unsigned long long a,
     const int cutest_assert_retval =                                    \
       assert_eq_comp((EXP), (REF))((EXP), (REF), cutest_o);             \
     if (0 != cutest_assert_retval) {                                    \
-      sprintf(cutest_stats.error_output,                                \
-              "%s %s:%d assert_eq(%s, " STR ") failed\n",               \
-              cutest_stats.error_output,                                \
+      char error_output_buf[1024];                                      \
+      sprintf(error_output_buf,                                         \
+              " %s:%d assert_eq(%s, " STR ") failed\n",                 \
               __FILE__,                                                 \
               __LINE__,                                                 \
               cutest_o);                                                \
+      strcat(cutest_stats.error_output, error_output_buf);              \
       cutest_assert_fail_cnt++;                                         \
     }                                                                   \
   }
@@ -647,12 +651,13 @@ int cutest_assert_eq_default(const unsigned long long a,
     const int cutest_assert_retval =                                    \
       assert_eq_comp((EXP), (REF))((EXP), (REF), cutest_o);             \
     if (0 != cutest_assert_retval) {                                    \
-      sprintf(cutest_stats.error_output,                                \
-              "%s %s:%d assert_eq(%s) failed\n",                        \
-              cutest_stats.error_output,                                \
+      char error_output_buf[1024];                                      \
+      sprintf(error_output_buf,                                         \
+              " %s:%d assert_eq(%s) failed\n",                          \
               __FILE__,                                                 \
               __LINE__,                                                 \
               cutest_o);                                                \
+      strcat(cutest_stats.error_output, error_output_buf);              \
       cutest_assert_fail_cnt++;                                         \
     }                                                                   \
   }
@@ -661,30 +666,36 @@ int cutest_assert_eq_default(const unsigned long long a,
 
 #define assert_eq_3(EXP, REF, STR)                                 \
   if ((EXP) != (REF)) {                                            \
-    sprintf(cutest_stats.error_output,                             \
-            "%s %s:%d assert_eq(" #EXP ", " #REF ", " STR ") "     \
-            "failed\n", cutest_stats.error_output,                 \
+    char error_output_buf[1024];                                   \
+    sprintf(error_output_buf,                                      \
+            " %s:%d assert_eq(" #EXP ", " #REF ", " STR ") "       \
+            "failed\n",                                            \
             __FILE__, __LINE__);                                   \
+    strcat(cutest_stats.error_output, error_output_buf);           \
     cutest_assert_fail_cnt++;                                      \
   }
 
 #define assert_eq_2(EXP, REF)                                      \
   if ((EXP) != (REF)) {                                            \
-    sprintf(cutest_stats.error_output,                             \
-            "%s %s:%d assert_eq(" #EXP ", " #REF ") "              \
-            "failed\n", cutest_stats.error_output,                 \
+    char error_output_buf[1024];                                   \
+    sprintf(error_output_buf,                                      \
+            " %s:%d assert_eq(" #EXP ", " #REF ") "                \
+            "failed\n",                                            \
             __FILE__, __LINE__);                                   \
+    strcat(cutest_stats.error_output, error_output_buf);           \
     cutest_assert_fail_cnt++;                                      \
   }
 
 #endif
 
 #define assert_eq_1(EXP)                                           \
   if (!(EXP)) {                                                    \
-    sprintf(cutest_stats.error_output,                             \
-            "%s %s:%d assert_eq(" #EXP ") "                        \
-            "failed\n", cutest_stats.error_output,                 \
+    char error_output_buf[1024];                                   \
+    sprintf(error_output_buf,                                      \
+            " %s:%d assert_eq(" #EXP ") "                          \
+            "failed\n",                                            \
             __FILE__, __LINE__);                                   \
+    strcat(cutest_stats.error_output, error_output_buf);           \
     cutest_assert_fail_cnt++;                                      \
   }
 
@@ -711,7 +722,7 @@ static int cutest_exit_code = EXIT_SUCCESS;
 struct {
   char suite_name[128];
   char design_under_test[128];
-  char error_output[1024*1024];
+  char error_output[1024*1024*10];
   int test_cnt;
   int fail_cnt;
   float elapsed_time;

diff --git a/src/cutest.mk b/src/cutest.mk
@@ -34,6 +34,9 @@ CUTEST_TEST_DIR ?=./
 
 # Some nice flags for compiling cutest-tests with good quality
 CUTEST_CFLAGS?=-g -pedantic -Wall -Wextra -std=c11
+# This makes valgrind work with long double values, should suffice for
+# most applications as well.
+CUTEST_CFLAGS+= -mlong-double-64
 
 ifneq (${LENIENT},0)
 	CUTEST_CFLAGS+=-D"CUTEST_LENIENT_ASSERTS=1"
@@ -151,7 +154,6 @@ check:: $(subst .c,,$(wildcard $(CUTEST_TEST_DIR)/*_test.c))
 memcheck:: $(subst .c,.memcheck,$(wildcard $(CUTEST_TEST_DIR)/*_test.c))
 
 valgrind:: $(subst .c,,$(wildcard $(CUTEST_TEST_DIR)/*_test.c))
-	echo $<
 	@R=true; \
 	processors=`cat /proc/cpuinfo | grep processor | wc -l`; \
 	for i in $^; do \